diff --git a/examples/ideogram4/model_training/lora/Ideogram-4-fp8.sh b/examples/ideogram4/model_training/lora/Ideogram-4-fp8.sh new file mode 100644 index 0000000000000000000000000000000000000000..17336229f1f063238748a7dc0b05d3f639cb2e5a --- /dev/null +++ b/examples/ideogram4/model_training/lora/Ideogram-4-fp8.sh @@ -0,0 +1,18 @@ +modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "ideogram4/Ideogram-4-fp8/*" --local_dir ./data/diffsynth_example_dataset + +accelerate launch examples/ideogram4/model_training/train.py \ + --learning_rate 1e-4 \ + --num_epochs 5 \ + --lora_rank 32 \ + --use_gradient_checkpointing \ + --find_unused_parameters \ + --dataset_base_path "./data/diffsynth_example_dataset/ideogram4/Ideogram-4-fp8" \ + --dataset_metadata_path "./data/diffsynth_example_dataset/ideogram4/Ideogram-4-fp8/metadata.json" \ + --model_id_with_origin_paths "ideogram-ai/ideogram-4-fp8:transformer/diffusion_pytorch_model.safetensors,ideogram-ai/ideogram-4-fp8:text_encoder/model.safetensors,ideogram-ai/ideogram-4-fp8:vae/diffusion_pytorch_model.safetensors" \ + --lora_base_model "dit" \ + --remove_prefix_in_ckpt "pipe.dit." \ + --max_pixels 1048576 \ + --dataset_repeat 100 \ + --output_path "./models/train/Ideogram-4-fp8_lora" \ + --lora_target_modules "attention.qkv,attention.o,feed_forward.w1,feed_forward.w2,feed_forward.w3,adaln_modulation" \ + --data_file_keys "image" diff --git a/examples/ideogram4/model_training/lora/Ideogram-4-nf4.sh b/examples/ideogram4/model_training/lora/Ideogram-4-nf4.sh new file mode 100644 index 0000000000000000000000000000000000000000..cc9104afb80eabd5f0c0ffb3f9c9c80d3e135112 --- /dev/null +++ b/examples/ideogram4/model_training/lora/Ideogram-4-nf4.sh @@ -0,0 +1,18 @@ +modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "ideogram4/Ideogram-4-nf4/*" --local_dir ./data/diffsynth_example_dataset + +accelerate launch examples/ideogram4/model_training/train.py \ + --learning_rate 1e-4 \ + --num_epochs 5 \ + --lora_rank 32 \ + --use_gradient_checkpointing \ + --find_unused_parameters \ + --dataset_base_path "./data/diffsynth_example_dataset/ideogram4/Ideogram-4-nf4" \ + --dataset_metadata_path "./data/diffsynth_example_dataset/ideogram4/Ideogram-4-nf4/metadata.json" \ + --model_id_with_origin_paths "ideogram-ai/ideogram-4-nf4:transformer/diffusion_pytorch_model.safetensors,ideogram-ai/ideogram-4-nf4:text_encoder/model.safetensors,ideogram-ai/ideogram-4-nf4:vae/diffusion_pytorch_model.safetensors" \ + --lora_base_model "dit" \ + --remove_prefix_in_ckpt "pipe.dit." \ + --max_pixels 1048576 \ + --dataset_repeat 100 \ + --output_path "./models/train/Ideogram-4-nf4_lora" \ + --lora_target_modules "attention.qkv,attention.o,feed_forward.w1,feed_forward.w2,feed_forward.w3,adaln_modulation" \ + --data_file_keys "image" diff --git a/examples/ideogram4/model_training/special/split_training/Ideogram-4-bf16-repackage.sh b/examples/ideogram4/model_training/special/split_training/Ideogram-4-bf16-repackage.sh new file mode 100644 index 0000000000000000000000000000000000000000..86f208048d621d9596ed66f12e665c352ed91f5a --- /dev/null +++ b/examples/ideogram4/model_training/special/split_training/Ideogram-4-bf16-repackage.sh @@ -0,0 +1,40 @@ +modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "ideogram4/Ideogram-4-bf16-repackage/*" --local_dir ./data/diffsynth_example_dataset + +# Stage 1: cache deterministic preprocessing outputs. +accelerate launch examples/ideogram4/model_training/train.py \ + --learning_rate 1e-4 \ + --num_epochs 5 \ + --lora_rank 32 \ + --use_gradient_checkpointing \ + --find_unused_parameters \ + --dataset_base_path ./data/diffsynth_example_dataset/ideogram4/Ideogram-4-bf16-repackage \ + --dataset_metadata_path ./data/diffsynth_example_dataset/ideogram4/Ideogram-4-bf16-repackage/metadata.json \ + --model_id_with_origin_paths DiffSynth-Studio/ideogram-4-bf16-repackage:transformer/diffusion_pytorch_model.safetensors,DiffSynth-Studio/ideogram-4-bf16-repackage:text_encoder/model.safetensors,DiffSynth-Studio/ideogram-4-bf16-repackage:vae/diffusion_pytorch_model.safetensors \ + --lora_base_model dit \ + --remove_prefix_in_ckpt pipe.dit. \ + --max_pixels 1048576 \ + --dataset_repeat 1 \ + --output_path ./models/train/Ideogram-4-bf16-repackage_split_cache \ + --lora_target_modules attention.qkv,attention.o,feed_forward.w1,feed_forward.w2,feed_forward.w3,adaln_modulation \ + --data_file_keys image \ + --offload_models DiffSynth-Studio/ideogram-4-bf16-repackage:transformer/diffusion_pytorch_model.safetensors \ + --task sft:data_process + +# Stage 2: train LoRA from the cached dataset. +accelerate launch examples/ideogram4/model_training/train.py \ + --learning_rate 1e-4 \ + --num_epochs 5 \ + --lora_rank 32 \ + --use_gradient_checkpointing \ + --find_unused_parameters \ + --dataset_base_path ./models/train/Ideogram-4-bf16-repackage_split_cache \ + --model_id_with_origin_paths DiffSynth-Studio/ideogram-4-bf16-repackage:transformer/diffusion_pytorch_model.safetensors,DiffSynth-Studio/ideogram-4-bf16-repackage:text_encoder/model.safetensors,DiffSynth-Studio/ideogram-4-bf16-repackage:vae/diffusion_pytorch_model.safetensors \ + --lora_base_model dit \ + --remove_prefix_in_ckpt pipe.dit. \ + --max_pixels 1048576 \ + --dataset_repeat 100 \ + --output_path ./models/train/Ideogram-4-bf16-repackage_split \ + --lora_target_modules attention.qkv,attention.o,feed_forward.w1,feed_forward.w2,feed_forward.w3,adaln_modulation \ + --data_file_keys image \ + --offload_models DiffSynth-Studio/ideogram-4-bf16-repackage:text_encoder/model.safetensors,DiffSynth-Studio/ideogram-4-bf16-repackage:vae/diffusion_pytorch_model.safetensors \ + --task sft:train diff --git a/examples/ideogram4/model_training/special/split_training/validate.py b/examples/ideogram4/model_training/special/split_training/validate.py new file mode 100644 index 0000000000000000000000000000000000000000..50977bdc7227614c4b1bca478b9ba3fe405a01ed --- /dev/null +++ b/examples/ideogram4/model_training/special/split_training/validate.py @@ -0,0 +1,21 @@ +from diffsynth.pipelines.ideogram4 import Ideogram4Pipeline +from diffsynth.core import ModelConfig +import torch + +pipe = Ideogram4Pipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="DiffSynth-Studio/ideogram-4-bf16-repackage", origin_file_pattern="transformer/diffusion_pytorch_model.safetensors"), + ModelConfig(model_id="DiffSynth-Studio/ideogram-4-bf16-repackage", origin_file_pattern="unconditional_transformer/diffusion_pytorch_model.safetensors"), + ModelConfig(model_id="DiffSynth-Studio/ideogram-4-bf16-repackage", origin_file_pattern="text_encoder/model.safetensors"), + ModelConfig(model_id="DiffSynth-Studio/ideogram-4-bf16-repackage", origin_file_pattern="vae/diffusion_pytorch_model.safetensors"), + ], + tokenizer_config=ModelConfig(model_id="ideogram-ai/ideogram-4-fp8", origin_file_pattern="tokenizer/"), +) +pipe.load_lora(pipe.dit, './models/train/Ideogram-4-bf16-repackage_split/epoch-4.safetensors', alpha=1) +# pipe.load_lora(pipe.dit_uncond, './models/train/Ideogram-4-bf16-repackage_split/epoch-4.safetensors', alpha=1) + +prompt = "{\"high_level_description\":\"A close-up photograph of a happy Pembroke Welsh Corgi sitting on a concrete wall, panting with its tongue out, set against a backdrop of blurred pink cherry blossoms and blue sky.\",\"style_description\":{\"aesthetics\":\"joyful, vibrant, spring-like, cute, energetic\",\"lighting\":\"bright natural daylight, soft diffuse sunlight, shallow depth of field\",\"photo\":\"85mm lens, f/2.0, bokeh background, sharp focus on dog face\",\"medium\":\"photograph\",\"color_palette\":[\"#E2725B\",\"#FFFFFF\",\"#FFB7C5\",\"#87CEEB\",\"#A9A9A9\"]},\"compositional_deconstruction\":{\"background\":\"Softly blurred background of pink cherry blossom branches against a pale blue sky. The bokeh effect creates a dreamy spring atmosphere. The background is out of focus to highlight the sharp details of the dog in the foreground.\",\"elements\":[{\"type\":\"obj\",\"bbox\":[150,200,900,850],\"desc\":\"A Pembroke Welsh Corgi with fluffy orange and white fur. Its mouth is open, panting with a pink tongue hanging out, expression is happy and excited. Ears are perked up. Sharp focus on the face and eyes.\"},{\"type\":\"obj\",\"bbox\":[850,0,1000,1000],\"desc\":\"A grey concrete wall or ledge at the bottom of the frame. The dog's front paws are resting near the edge. Rough texture.\"}]}}" +image = pipe(prompt=prompt, height=1024, width=1024, num_inference_steps=48, cfg_scale=7.0, seed=0) +image.save('split_training_Ideogram-4.jpg') diff --git a/examples/ideogram4/model_training/train.py b/examples/ideogram4/model_training/train.py new file mode 100644 index 0000000000000000000000000000000000000000..69d8d56142248d1285d6eb57a341b7e9e21764f2 --- /dev/null +++ b/examples/ideogram4/model_training/train.py @@ -0,0 +1,132 @@ +import torch, os, argparse, accelerate +from diffsynth.core import UnifiedDataset +from diffsynth.pipelines.ideogram4 import Ideogram4Pipeline, ModelConfig +from diffsynth.diffusion import * +os.environ["TOKENIZERS_PARALLELISM"] = "false" + + +class Ideogram4TrainingModule(DiffusionTrainingModule): + def __init__( + self, + model_paths=None, model_id_with_origin_paths=None, + tokenizer_path=None, + trainable_models=None, + lora_base_model=None, lora_target_modules="", lora_rank=32, lora_checkpoint=None, + preset_lora_path=None, preset_lora_model=None, + use_gradient_checkpointing=True, + use_gradient_checkpointing_offload=False, + extra_inputs=None, + fp8_models=None, + offload_models=None, + quant_options=None, + device="cpu", + task="sft", + ): + super().__init__() + model_configs = self.parse_model_configs(model_paths, model_id_with_origin_paths, fp8_models=fp8_models, offload_models=offload_models, quant_options=quant_options, device=device) + tokenizer_config = self.parse_path_or_model_id(tokenizer_path, default_value=ModelConfig(model_id="ideogram-ai/ideogram-4-fp8", origin_file_pattern="tokenizer/")) + self.pipe = Ideogram4Pipeline.from_pretrained(torch_dtype=torch.bfloat16, device=device, model_configs=model_configs, tokenizer_config=tokenizer_config) + self.pipe = self.split_pipeline_units(task, self.pipe, trainable_models, lora_base_model) + + self.switch_pipe_to_training_mode( + self.pipe, trainable_models, + lora_base_model, lora_target_modules, lora_rank, lora_checkpoint, + preset_lora_path, preset_lora_model, + task=task, + ) + + self.use_gradient_checkpointing = use_gradient_checkpointing + self.use_gradient_checkpointing_offload = use_gradient_checkpointing_offload + self.extra_inputs = extra_inputs.split(",") if extra_inputs is not None else [] + self.fp8_models = fp8_models + self.task = task + self.task_to_loss = { + "sft:data_process": lambda pipe, *args: args, + "sft": lambda pipe, inputs_shared, inputs_posi, inputs_nega: FlowMatchSFTLoss(pipe, **inputs_shared, **inputs_posi), + "sft:train": lambda pipe, inputs_shared, inputs_posi, inputs_nega: FlowMatchSFTLoss(pipe, **inputs_shared, **inputs_posi), + } + + def get_pipeline_inputs(self, data): + inputs_posi = {"prompt": data["prompt"]} + inputs_nega = {"negative_prompt": ""} + inputs_shared = { + "input_image": data["image"], + "height": data["image"].size[1], + "width": data["image"].size[0], + "cfg_scale": 1, + "rand_device": self.pipe.device, + "use_gradient_checkpointing": self.use_gradient_checkpointing, + "use_gradient_checkpointing_offload": self.use_gradient_checkpointing_offload, + } + inputs_shared = self.parse_extra_inputs(data, self.extra_inputs, inputs_shared) + return inputs_shared, inputs_posi, inputs_nega + + def forward(self, data, inputs=None): + if inputs is None: inputs = self.get_pipeline_inputs(data) + inputs = self.transfer_data_to_device(inputs, self.pipe.device, self.pipe.torch_dtype) + for unit in self.pipe.units: + inputs = self.pipe.unit_runner(unit, self.pipe, *inputs) + loss = self.task_to_loss[self.task](self.pipe, *inputs) + return loss + + +def ideogram4_parser(): + parser = argparse.ArgumentParser(description="Ideogram4 training.") + parser = add_general_config(parser) + parser = add_image_size_config(parser) + parser.add_argument("--tokenizer_path", type=str, default=None, help="Path to tokenizer.") + return parser + + +if __name__ == "__main__": + parser = ideogram4_parser() + args = parser.parse_args() + accelerator = accelerate.Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + kwargs_handlers=[accelerate.DistributedDataParallelKwargs(find_unused_parameters=args.find_unused_parameters)], + ) + dataset = UnifiedDataset( + base_path=args.dataset_base_path, + metadata_path=args.dataset_metadata_path, + repeat=args.dataset_repeat, + data_file_keys=args.data_file_keys.split(","), + main_data_operator=UnifiedDataset.default_image_operator( + base_path=args.dataset_base_path, + max_pixels=args.max_pixels, + height=args.height, + width=args.width, + height_division_factor=16, + width_division_factor=16, + ), + ) + model = Ideogram4TrainingModule( + model_paths=args.model_paths, + model_id_with_origin_paths=args.model_id_with_origin_paths, + tokenizer_path=args.tokenizer_path, + trainable_models=args.trainable_models, + lora_base_model=args.lora_base_model, + lora_target_modules=args.lora_target_modules, + lora_rank=args.lora_rank, + lora_checkpoint=args.lora_checkpoint, + preset_lora_path=args.preset_lora_path, + preset_lora_model=args.preset_lora_model, + use_gradient_checkpointing=args.use_gradient_checkpointing, + use_gradient_checkpointing_offload=args.use_gradient_checkpointing_offload, + extra_inputs=args.extra_inputs, + fp8_models=args.fp8_models, + offload_models=args.offload_models, + quant_options=args.quant_options, + task=args.task, + device=accelerator.device, + ) + model_logger = ModelLogger( + args.output_path, + remove_prefix_in_ckpt=args.remove_prefix_in_ckpt, + enable_csv_log=args.enable_csv_log, + ) + launcher_map = { + "sft:data_process": launch_data_process_task, + "sft": launch_training_task, + "sft:train": launch_training_task, + } + launcher_map[args.task](accelerator, dataset, model, model_logger, args=args) diff --git a/examples/ideogram4/model_training/validate_lora/Ideogram-4-bf16-repackage.py b/examples/ideogram4/model_training/validate_lora/Ideogram-4-bf16-repackage.py new file mode 100644 index 0000000000000000000000000000000000000000..93638fa12c8f832b756a9872a19c86314fa41df5 --- /dev/null +++ b/examples/ideogram4/model_training/validate_lora/Ideogram-4-bf16-repackage.py @@ -0,0 +1,21 @@ +from diffsynth.pipelines.ideogram4 import Ideogram4Pipeline +from diffsynth.core import ModelConfig +import torch + +pipe = Ideogram4Pipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="DiffSynth-Studio/ideogram-4-bf16-repackage", origin_file_pattern="transformer/diffusion_pytorch_model.safetensors"), + ModelConfig(model_id="DiffSynth-Studio/ideogram-4-bf16-repackage", origin_file_pattern="unconditional_transformer/diffusion_pytorch_model.safetensors"), + ModelConfig(model_id="DiffSynth-Studio/ideogram-4-bf16-repackage", origin_file_pattern="text_encoder/model.safetensors"), + ModelConfig(model_id="DiffSynth-Studio/ideogram-4-bf16-repackage", origin_file_pattern="vae/diffusion_pytorch_model.safetensors"), + ], + tokenizer_config=ModelConfig(model_id="ideogram-ai/ideogram-4-fp8", origin_file_pattern="tokenizer/"), +) +pipe.load_lora(pipe.dit, "models/train/Ideogram-4-bf16-repackage_lora/epoch-4.safetensors", alpha=1) +pipe.load_lora(pipe.dit_uncond, "models/train/Ideogram-4-bf16-repackage_lora/epoch-4.safetensors", alpha=1) + +prompt = "{\"high_level_description\":\"A close-up photograph of a happy Pembroke Welsh Corgi sitting on a concrete wall, panting with its tongue out, set against a backdrop of blurred pink cherry blossoms and blue sky.\",\"style_description\":{\"aesthetics\":\"joyful, vibrant, spring-like, cute, energetic\",\"lighting\":\"bright natural daylight, soft diffuse sunlight, shallow depth of field\",\"photo\":\"85mm lens, f/2.0, bokeh background, sharp focus on dog face\",\"medium\":\"photograph\",\"color_palette\":[\"#E2725B\",\"#FFFFFF\",\"#FFB7C5\",\"#87CEEB\",\"#A9A9A9\"]},\"compositional_deconstruction\":{\"background\":\"Softly blurred background of pink cherry blossom branches against a pale blue sky. The bokeh effect creates a dreamy spring atmosphere. The background is out of focus to highlight the sharp details of the dog in the foreground.\",\"elements\":[{\"type\":\"obj\",\"bbox\":[150,200,900,850],\"desc\":\"A Pembroke Welsh Corgi with fluffy orange and white fur. Its mouth is open, panting with a pink tongue hanging out, expression is happy and excited. Ears are perked up. Sharp focus on the face and eyes.\"},{\"type\":\"obj\",\"bbox\":[850,0,1000,1000],\"desc\":\"A grey concrete wall or ledge at the bottom of the frame. The dog's front paws are resting near the edge. Rough texture.\"}]}}" +image = pipe(prompt=prompt, height=1024, width=1024, num_inference_steps=48, cfg_scale=7.0, seed=0) +image.save("image_lora.jpg") diff --git a/examples/ideogram4/model_training/validate_lora/Ideogram-4-fp8.py b/examples/ideogram4/model_training/validate_lora/Ideogram-4-fp8.py new file mode 100644 index 0000000000000000000000000000000000000000..8b338939b9d7d867e702f3a7e159a27b7fdca2c7 --- /dev/null +++ b/examples/ideogram4/model_training/validate_lora/Ideogram-4-fp8.py @@ -0,0 +1,32 @@ +from diffsynth.pipelines.ideogram4 import Ideogram4Pipeline +from diffsynth.core import ModelConfig +import torch + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cpu", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = Ideogram4Pipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="ideogram-ai/ideogram-4-fp8", origin_file_pattern="transformer/diffusion_pytorch_model.safetensors", **vram_config), + ModelConfig(model_id="ideogram-ai/ideogram-4-fp8", origin_file_pattern="unconditional_transformer/diffusion_pytorch_model.safetensors", **vram_config), + ModelConfig(model_id="ideogram-ai/ideogram-4-fp8", origin_file_pattern="text_encoder/model.safetensors", **vram_config), + ModelConfig(model_id="ideogram-ai/ideogram-4-fp8", origin_file_pattern="vae/diffusion_pytorch_model.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="ideogram-ai/ideogram-4-fp8", origin_file_pattern="tokenizer/"), + vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 0.5, +) +pipe.load_lora(pipe.dit, "models/train/Ideogram-4-fp8_lora/epoch-4.safetensors", alpha=1) +pipe.load_lora(pipe.dit_uncond, "models/train/Ideogram-4-fp8_lora/epoch-4.safetensors", alpha=1) + +prompt = "{\"high_level_description\":\"A close-up photograph of a happy Pembroke Welsh Corgi sitting on a concrete wall, panting with its tongue out, set against a backdrop of blurred pink cherry blossoms and blue sky.\",\"style_description\":{\"aesthetics\":\"joyful, vibrant, spring-like, cute, energetic\",\"lighting\":\"bright natural daylight, soft diffuse sunlight, shallow depth of field\",\"photo\":\"85mm lens, f/2.0, bokeh background, sharp focus on dog face\",\"medium\":\"photograph\",\"color_palette\":[\"#E2725B\",\"#FFFFFF\",\"#FFB7C5\",\"#87CEEB\",\"#A9A9A9\"]},\"compositional_deconstruction\":{\"background\":\"Softly blurred background of pink cherry blossom branches against a pale blue sky. The bokeh effect creates a dreamy spring atmosphere. The background is out of focus to highlight the sharp details of the dog in the foreground.\",\"elements\":[{\"type\":\"obj\",\"bbox\":[150,200,900,850],\"desc\":\"A Pembroke Welsh Corgi with fluffy orange and white fur. Its mouth is open, panting with a pink tongue hanging out, expression is happy and excited. Ears are perked up. Sharp focus on the face and eyes.\"},{\"type\":\"obj\",\"bbox\":[850,0,1000,1000],\"desc\":\"A grey concrete wall or ledge at the bottom of the frame. The dog's front paws are resting near the edge. Rough texture.\"}]}}" +image = pipe(prompt=prompt, height=1024, width=1024, num_inference_steps=48, cfg_scale=7.0, seed=0) +image.save("image_lora.jpg") diff --git a/examples/ideogram4/model_training/validate_lora/Ideogram-4-nf4.py b/examples/ideogram4/model_training/validate_lora/Ideogram-4-nf4.py new file mode 100644 index 0000000000000000000000000000000000000000..6bc13309f1d8e3f03c428a34241259607d7cd8fe --- /dev/null +++ b/examples/ideogram4/model_training/validate_lora/Ideogram-4-nf4.py @@ -0,0 +1,32 @@ +from diffsynth.pipelines.ideogram4 import Ideogram4Pipeline +from diffsynth.core import ModelConfig +import torch + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cpu", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = Ideogram4Pipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="ideogram-ai/ideogram-4-nf4", origin_file_pattern="transformer/diffusion_pytorch_model.safetensors", **vram_config), + ModelConfig(model_id="ideogram-ai/ideogram-4-nf4", origin_file_pattern="unconditional_transformer/diffusion_pytorch_model.safetensors", **vram_config), + ModelConfig(model_id="ideogram-ai/ideogram-4-nf4", origin_file_pattern="text_encoder/model.safetensors", **vram_config), + ModelConfig(model_id="ideogram-ai/ideogram-4-nf4", origin_file_pattern="vae/diffusion_pytorch_model.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="ideogram-ai/ideogram-4-nf4", origin_file_pattern="tokenizer/"), + vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 0.5, +) +pipe.load_lora(pipe.dit, "models/train/Ideogram-4-nf4_lora/epoch-4.safetensors", alpha=1) +pipe.load_lora(pipe.dit_uncond, "models/train/Ideogram-4-nf4_lora/epoch-4.safetensors", alpha=1) + +prompt = "{\"high_level_description\":\"A close-up photograph of a happy Pembroke Welsh Corgi sitting on a concrete wall, panting with its tongue out, set against a backdrop of blurred pink cherry blossoms and blue sky.\",\"style_description\":{\"aesthetics\":\"joyful, vibrant, spring-like, cute, energetic\",\"lighting\":\"bright natural daylight, soft diffuse sunlight, shallow depth of field\",\"photo\":\"85mm lens, f/2.0, bokeh background, sharp focus on dog face\",\"medium\":\"photograph\",\"color_palette\":[\"#E2725B\",\"#FFFFFF\",\"#FFB7C5\",\"#87CEEB\",\"#A9A9A9\"]},\"compositional_deconstruction\":{\"background\":\"Softly blurred background of pink cherry blossom branches against a pale blue sky. The bokeh effect creates a dreamy spring atmosphere. The background is out of focus to highlight the sharp details of the dog in the foreground.\",\"elements\":[{\"type\":\"obj\",\"bbox\":[150,200,900,850],\"desc\":\"A Pembroke Welsh Corgi with fluffy orange and white fur. Its mouth is open, panting with a pink tongue hanging out, expression is happy and excited. Ears are perked up. Sharp focus on the face and eyes.\"},{\"type\":\"obj\",\"bbox\":[850,0,1000,1000],\"desc\":\"A grey concrete wall or ledge at the bottom of the frame. The dog's front paws are resting near the edge. Rough texture.\"}]}}" +image = pipe(prompt=prompt, height=1024, width=1024, num_inference_steps=48, cfg_scale=7.0, seed=0) +image.save("image_lora.jpg") diff --git a/examples/image_quality_metric/aesthetic.py b/examples/image_quality_metric/aesthetic.py new file mode 100644 index 0000000000000000000000000000000000000000..4a4f432ad506524f000d663f4f00e3c2bc463180 --- /dev/null +++ b/examples/image_quality_metric/aesthetic.py @@ -0,0 +1,16 @@ +from diffsynth.metrics import AestheticMetric, ModelConfig +from modelscope import dataset_snapshot_download +from PIL import Image + +dataset_snapshot_download( + "DiffSynth-Studio/diffsynth_example_dataset", + allow_file_pattern="flux/FLUX.1-dev/*", + local_dir="./data/diffsynth_example_dataset", +) +image = Image.open("data/diffsynth_example_dataset/flux/FLUX.1-dev/1.jpg").convert("RGB") +metric = AestheticMetric.from_pretrained( + model_config=ModelConfig(model_id="DiffSynth-Studio/ImageMetrics", origin_file_pattern="Aesthetic/model.safetensors"), + device="cuda" +) +score = metric.compute(image)[0] +print(f"Aesthetic score: {score:.3f}") diff --git a/examples/image_quality_metric/bioclip.py b/examples/image_quality_metric/bioclip.py new file mode 100644 index 0000000000000000000000000000000000000000..d95d1d7b97d9909a29294a68f57beed73be4774d --- /dev/null +++ b/examples/image_quality_metric/bioclip.py @@ -0,0 +1,17 @@ +from diffsynth.metrics import BioCLIPMetric, ModelConfig +from modelscope import dataset_snapshot_download +from PIL import Image + +dataset_snapshot_download( + "DiffSynth-Studio/diffsynth_example_dataset", + allow_file_pattern="flux/FLUX.1-dev/*", + local_dir="./data/diffsynth_example_dataset", +) +image = Image.open("data/diffsynth_example_dataset/flux/FLUX.1-dev/1.jpg").convert("RGB") +prompt = "a photo of Animalia Chordata Mammalia Carnivora Canidae Canis Canis lupus familiaris with common name domestic dog." +metric = BioCLIPMetric.from_pretrained( + model_config=ModelConfig(model_id="DiffSynth-Studio/ImageMetrics", origin_file_pattern="BioCLIPv2/open_clip_model.safetensors"), + device="cuda", +) +score = metric.compute(prompt, image)[0] +print(f"BioCLIP score: {score:.3f}") diff --git a/examples/image_quality_metric/clipscore.py b/examples/image_quality_metric/clipscore.py new file mode 100644 index 0000000000000000000000000000000000000000..c894bffbf2456a3759f5588a937be0bf9b6e2b54 --- /dev/null +++ b/examples/image_quality_metric/clipscore.py @@ -0,0 +1,17 @@ +from diffsynth.metrics import CLIPMetric, ModelConfig +from modelscope import dataset_snapshot_download +from PIL import Image + +dataset_snapshot_download( + "DiffSynth-Studio/diffsynth_example_dataset", + allow_file_pattern="flux/FLUX.1-dev/*", + local_dir="./data/diffsynth_example_dataset", +) +image = Image.open("data/diffsynth_example_dataset/flux/FLUX.1-dev/1.jpg").convert("RGB") +prompt = "a dog" +metric = CLIPMetric.from_pretrained( + model_config=ModelConfig(model_id="DiffSynth-Studio/ImageMetrics", origin_file_pattern="CLIP-ViT-H-14-laion2B-s32B-b79K/model.safetensors"), + device="cuda" +) +score = metric.compute(prompt, image)[0] +print(f"CLIP score: {score:.3f}") \ No newline at end of file diff --git a/examples/image_quality_metric/fid.py b/examples/image_quality_metric/fid.py new file mode 100644 index 0000000000000000000000000000000000000000..7a1ed74cc45928a5bb74febc3c5930d4e559bdb7 --- /dev/null +++ b/examples/image_quality_metric/fid.py @@ -0,0 +1,17 @@ +from diffsynth.metrics import FIDMetric, ModelConfig +from modelscope import dataset_snapshot_download + +dataset_snapshot_download( + "DiffSynth-Studio/diffsynth_example_dataset", + allow_file_pattern=["flux/FLUX.1-dev/*", "flux2/FLUX.2-dev/*"], + local_dir="./data/diffsynth_example_dataset", +) +metric = FIDMetric.from_pretrained( + model_config=ModelConfig(model_id="DiffSynth-Studio/ImageMetrics", origin_file_pattern="FID/model.safetensors"), + device="cuda", +) +score = metric.compute( + "./data/diffsynth_example_dataset/flux/FLUX.1-dev", + "./data/diffsynth_example_dataset/flux2/FLUX.2-dev", +) +print(f"FID score: {score:.3f}") \ No newline at end of file diff --git a/examples/image_quality_metric/hpsv2.py b/examples/image_quality_metric/hpsv2.py new file mode 100644 index 0000000000000000000000000000000000000000..fd6a18c17205cd8915288c8c6d8436347f055365 --- /dev/null +++ b/examples/image_quality_metric/hpsv2.py @@ -0,0 +1,17 @@ +from diffsynth.metrics import HPSv2Metric, ModelConfig +from modelscope import dataset_snapshot_download +from PIL import Image + +dataset_snapshot_download( + "DiffSynth-Studio/diffsynth_example_dataset", + allow_file_pattern="flux/FLUX.1-dev/*", + local_dir="./data/diffsynth_example_dataset", +) +image = Image.open("data/diffsynth_example_dataset/flux/FLUX.1-dev/1.jpg").convert("RGB") +prompt = "a dog" +metric = HPSv2Metric.from_pretrained( + model_config=ModelConfig(model_id="DiffSynth-Studio/ImageMetrics", origin_file_pattern="HPSv2/model.safetensors"), + device="cuda" +) +score = metric.compute(prompt, image)[0] +print(f"HPSv2 score: {score:.3f}") \ No newline at end of file diff --git a/examples/image_quality_metric/hpsv3.py b/examples/image_quality_metric/hpsv3.py new file mode 100644 index 0000000000000000000000000000000000000000..42cd967ba6ef916264600c2fb1c8dd4bfb521619 --- /dev/null +++ b/examples/image_quality_metric/hpsv3.py @@ -0,0 +1,17 @@ +from diffsynth.metrics import HPSv3Metric, ModelConfig +from modelscope import dataset_snapshot_download +from PIL import Image + +dataset_snapshot_download( + "DiffSynth-Studio/diffsynth_example_dataset", + allow_file_pattern="flux/FLUX.1-dev/*", + local_dir="./data/diffsynth_example_dataset", +) +image = Image.open("data/diffsynth_example_dataset/flux/FLUX.1-dev/1.jpg").convert("RGB") +prompt = "a dog" +metric = HPSv3Metric.from_pretrained( + model_config=ModelConfig(model_id="DiffSynth-Studio/ImageMetrics", origin_file_pattern="HPSv3/model.safetensors"), + device="cuda" +) +score = metric.compute(prompt, image)[0] +print(f"HPSv3 score: {score:.3f}") \ No newline at end of file diff --git a/examples/image_quality_metric/image_reward.py b/examples/image_quality_metric/image_reward.py new file mode 100644 index 0000000000000000000000000000000000000000..ac16acd98d72993494e9f20c5129e38f4f6dcb9d --- /dev/null +++ b/examples/image_quality_metric/image_reward.py @@ -0,0 +1,17 @@ +from diffsynth.metrics import ImageRewardMetric, ModelConfig +from modelscope import dataset_snapshot_download +from PIL import Image + +dataset_snapshot_download( + "DiffSynth-Studio/diffsynth_example_dataset", + allow_file_pattern="flux/FLUX.1-dev/*", + local_dir="./data/diffsynth_example_dataset", +) +image = Image.open("data/diffsynth_example_dataset/flux/FLUX.1-dev/1.jpg").convert("RGB") +prompt = "a dog" +metric = ImageRewardMetric.from_pretrained( + model_config=ModelConfig(model_id="DiffSynth-Studio/ImageMetrics", origin_file_pattern="ImageReward/model.safetensors"), + device="cuda" +) +score = metric.compute(prompt, image)[0] +print(f"ImageReward score: {score:.3f}") \ No newline at end of file diff --git a/examples/image_quality_metric/lpips.py b/examples/image_quality_metric/lpips.py new file mode 100644 index 0000000000000000000000000000000000000000..6e15b5a91ffe351764397336a6f962bd7d4d9383 --- /dev/null +++ b/examples/image_quality_metric/lpips.py @@ -0,0 +1,33 @@ +from diffsynth.metrics import LPIPSMetric, ModelConfig +from modelscope import dataset_snapshot_download + +dataset_snapshot_download( + "DiffSynth-Studio/diffsynth_example_dataset", + allow_file_pattern=["flux/FLUX.1-dev/*", "flux2/FLUX.2-dev/*"], + local_dir="./data/diffsynth_example_dataset", +) + +# net="alex" with LPIPS/alexnet.safetensors (default) +# For VGG: net="vgg", model_config=ModelConfig(model_id="DiffSynth-Studio/ImageMetrics", origin_file_pattern="LPIPS/vgg.safetensors") +# For SqueezeNet: net="squeeze", model_config=ModelConfig(model_id="DiffSynth-Studio/ImageMetrics", origin_file_pattern="LPIPS/squeezenet.safetensors") +metric = LPIPSMetric.from_pretrained( + net="alex", + model_config=ModelConfig( + model_id="DiffSynth-Studio/ImageMetrics", + origin_file_pattern="LPIPS/alexnet.safetensors", + ), + device="cuda", + target_size=512, +) + +score = metric.compute( + "./data/diffsynth_example_dataset/flux/FLUX.1-dev/1.jpg", + "./data/diffsynth_example_dataset/flux/FLUX.1-dev/2.jpg", +) +print(f"LPIPS score (image vs image): {score:.4f}") + +score = metric.compute( + "./data/diffsynth_example_dataset/flux/FLUX.1-dev", + "./data/diffsynth_example_dataset/flux2/FLUX.2-dev", +) +print(f"LPIPS score (dir vs dir): {score:.4f}") diff --git a/examples/image_quality_metric/pickscore.py b/examples/image_quality_metric/pickscore.py new file mode 100644 index 0000000000000000000000000000000000000000..6f93effd8730b62deb9531560f57aa387bb09932 --- /dev/null +++ b/examples/image_quality_metric/pickscore.py @@ -0,0 +1,17 @@ +from diffsynth.metrics import PickScoreMetric, ModelConfig +from modelscope import dataset_snapshot_download +from PIL import Image + +dataset_snapshot_download( + "DiffSynth-Studio/diffsynth_example_dataset", + allow_file_pattern="flux/FLUX.1-dev/*", + local_dir="./data/diffsynth_example_dataset", +) +image = Image.open("data/diffsynth_example_dataset/flux/FLUX.1-dev/1.jpg").convert("RGB") +prompt = "a dog" +metric = PickScoreMetric.from_pretrained( + model_config=ModelConfig(model_id="DiffSynth-Studio/ImageMetrics", origin_file_pattern="PickScore/model.safetensors"), + device="cuda" +) +score = metric.compute(prompt, image)[0] +print(f"PickScore score:: {score:.3f}") \ No newline at end of file diff --git a/examples/image_quality_metric/qwen_image_bench.py b/examples/image_quality_metric/qwen_image_bench.py new file mode 100644 index 0000000000000000000000000000000000000000..5b2e813dc485c3ddd66b2112c097ab9900b0e1b0 --- /dev/null +++ b/examples/image_quality_metric/qwen_image_bench.py @@ -0,0 +1,21 @@ +from diffsynth.metrics import ModelConfig, QwenImageBenchMetric +from modelscope import dataset_snapshot_download +from PIL import Image +import json + +dataset_snapshot_download( + "DiffSynth-Studio/diffsynth_example_dataset", + allow_file_pattern="flux/FLUX.1-dev/*", + local_dir="./data/diffsynth_example_dataset", +) +image = Image.open("data/diffsynth_example_dataset/flux/FLUX.1-dev/1.jpg").convert("RGB") +prompt = "a dog" +metric = QwenImageBenchMetric.from_pretrained( + model_config=ModelConfig(model_id="Qwen/Qwen-Image-Bench", origin_file_pattern="model-*.safetensors"), + processor_config=ModelConfig(model_id="Qwen/Qwen-Image-Bench"), + device="cuda", +) +details = metric.evaluate(prompt, image)[0] +print(json.dumps(details["level1_scores"], indent=4, ensure_ascii=False)) +print(json.dumps(details["level2_scores"], indent=4, ensure_ascii=False)) +print(json.dumps(details["level3_scores"], indent=4, ensure_ascii=False)) diff --git a/examples/image_quality_metric/unified_reward_2.py b/examples/image_quality_metric/unified_reward_2.py new file mode 100644 index 0000000000000000000000000000000000000000..af31c9672daace674c7d0e18cced48a94abb1bbf --- /dev/null +++ b/examples/image_quality_metric/unified_reward_2.py @@ -0,0 +1,19 @@ +from diffsynth.metrics import ModelConfig, UnifiedReward2Metric +from modelscope import dataset_snapshot_download +from PIL import Image +import json + +dataset_snapshot_download( + "DiffSynth-Studio/diffsynth_example_dataset", + allow_file_pattern="flux/FLUX.1-dev/*", + local_dir="./data/diffsynth_example_dataset", +) +image = Image.open("data/diffsynth_example_dataset/flux/FLUX.1-dev/1.jpg").convert("RGB") +prompt = "a dog" +metric = UnifiedReward2Metric.from_pretrained( + model_config=ModelConfig(model_id="DiffSynth-Studio/ImageMetrics", origin_file_pattern="UnifiedReward-2.0-qwen35-9b/model-*.safetensors"), + processor_config=ModelConfig(model_id="DiffSynth-Studio/ImageMetrics", origin_file_pattern="UnifiedReward-2.0-qwen35-9b/"), + device="cuda", +) +details = metric.evaluate(prompt, image)[0] +print(json.dumps(details, indent=4, ensure_ascii=False)) diff --git a/examples/image_quality_metric/unified_reward_edit.py b/examples/image_quality_metric/unified_reward_edit.py new file mode 100644 index 0000000000000000000000000000000000000000..5324d563d68ee486880bd42051e21c7016f8b437 --- /dev/null +++ b/examples/image_quality_metric/unified_reward_edit.py @@ -0,0 +1,25 @@ +from diffsynth.metrics import ModelConfig, UnifiedRewardEditMetric +from modelscope import dataset_snapshot_download +from PIL import Image +import json + +dataset_snapshot_download( + "DiffSynth-Studio/diffsynth_example_dataset", + allow_file_pattern="qwen_image/Qwen-Image-Edit/*", + local_dir="./data/diffsynth_example_dataset", +) +source_image = Image.open("data/diffsynth_example_dataset/qwen_image/Qwen-Image-Edit/edit/image1.jpg").convert("RGB") +edited_image_1 = Image.open("data/diffsynth_example_dataset/qwen_image/Qwen-Image-Edit/edit/image2.jpg").convert("RGB") +edited_image_2 = source_image +instruction = "Change the dress to pink." +metric = UnifiedRewardEditMetric.from_pretrained( + model_config=ModelConfig(model_id="DiffSynth-Studio/ImageMetrics", origin_file_pattern="UnifiedReward-Edit-qwen3vl-8b/model-*.safetensors"), + processor_config=ModelConfig(model_id="DiffSynth-Studio/ImageMetrics", origin_file_pattern="UnifiedReward-Edit-qwen3vl-8b/"), + device="cuda", +) +details = metric.evaluate(instruction, [source_image, edited_image_1], task="edit_pointwise_score")[0] +print(json.dumps(details, indent=4, ensure_ascii=False)) +details = metric.evaluate(instruction, [source_image, edited_image_1, edited_image_2], task="edit_pairwise_rank")[0] +print(json.dumps(details, indent=4, ensure_ascii=False)) +details = metric.evaluate(instruction, [source_image, edited_image_1, edited_image_2], task="edit_pairwise_score")[0] +print(json.dumps(details, indent=4, ensure_ascii=False)) diff --git a/examples/joyai_image/model_inference/JoyAI-Image-Edit.py b/examples/joyai_image/model_inference/JoyAI-Image-Edit.py new file mode 100644 index 0000000000000000000000000000000000000000..a8aa0d7dc0ab7d3069de3d7aef901c86817f5d61 --- /dev/null +++ b/examples/joyai_image/model_inference/JoyAI-Image-Edit.py @@ -0,0 +1,39 @@ +from diffsynth.pipelines.joyai_image import JoyAIImagePipeline, ModelConfig +import torch +from PIL import Image +from modelscope import dataset_snapshot_download + +# Download dataset +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/diffsynth_example_dataset", + local_dir="data/diffsynth_example_dataset", + allow_file_pattern="joyai_image/JoyAI-Image-Edit/*" +) + +pipe = JoyAIImagePipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="jd-opensource/JoyAI-Image-Edit", origin_file_pattern="transformer/transformer.pth"), + ModelConfig(model_id="jd-opensource/JoyAI-Image-Edit", origin_file_pattern="JoyAI-Image-Und/model*.safetensors"), + ModelConfig(model_id="jd-opensource/JoyAI-Image-Edit", origin_file_pattern="vae/Wan2.1_VAE.pth"), + ], + processor_config=ModelConfig(model_id="jd-opensource/JoyAI-Image-Edit", origin_file_pattern="JoyAI-Image-Und/"), +) + +# Use first sample from dataset +dataset_base_path = "data/diffsynth_example_dataset/joyai_image/JoyAI-Image-Edit" +prompt = "将裙子改为粉色" +edit_image = Image.open(f"{dataset_base_path}/edit/image1.jpg").convert("RGB") + +output = pipe( + prompt=prompt, + edit_image=edit_image, + height=1024, + width=1024, + seed=1, + num_inference_steps=30, + cfg_scale=5.0, +) + +output.save("output_joyai_edit.png") diff --git a/examples/joyai_image/model_inference_low_vram/JoyAI-Image-Edit.py b/examples/joyai_image/model_inference_low_vram/JoyAI-Image-Edit.py new file mode 100644 index 0000000000000000000000000000000000000000..d6a67bcc9c116088ee180766bef974c2e38de1a0 --- /dev/null +++ b/examples/joyai_image/model_inference_low_vram/JoyAI-Image-Edit.py @@ -0,0 +1,51 @@ +from diffsynth.pipelines.joyai_image import JoyAIImagePipeline, ModelConfig +import torch +from PIL import Image +from modelscope import dataset_snapshot_download + +# Download dataset +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/diffsynth_example_dataset", + local_dir="data/diffsynth_example_dataset", + allow_file_pattern="joyai_image/JoyAI-Image-Edit/*" +) + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cpu", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} + +pipe = JoyAIImagePipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="jd-opensource/JoyAI-Image-Edit", origin_file_pattern="transformer/transformer.pth", **vram_config), + ModelConfig(model_id="jd-opensource/JoyAI-Image-Edit", origin_file_pattern="JoyAI-Image-Und/model*.safetensors", **vram_config), + ModelConfig(model_id="jd-opensource/JoyAI-Image-Edit", origin_file_pattern="vae/Wan2.1_VAE.pth", **vram_config), + ], + processor_config=ModelConfig(model_id="jd-opensource/JoyAI-Image-Edit", origin_file_pattern="JoyAI-Image-Und/"), + vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 0.5, +) + +# Use first sample from dataset +dataset_base_path = "data/diffsynth_example_dataset/joyai_image/JoyAI-Image-Edit" +prompt = "将裙子改为粉色" +edit_image = Image.open(f"{dataset_base_path}/edit/image1.jpg").convert("RGB") + +output = pipe( + prompt=prompt, + edit_image=edit_image, + height=1024, + width=1024, + seed=0, + num_inference_steps=30, + cfg_scale=5.0, +) + +output.save("output_joyai_edit_low_vram.png") diff --git a/examples/joyai_image/model_training/full/JoyAI-Image-Edit.sh b/examples/joyai_image/model_training/full/JoyAI-Image-Edit.sh new file mode 100644 index 0000000000000000000000000000000000000000..1af9f18297c66991aeebe74c88467f48b0c449aa --- /dev/null +++ b/examples/joyai_image/model_training/full/JoyAI-Image-Edit.sh @@ -0,0 +1,35 @@ +# Dataset: data/diffsynth_example_dataset/joyai_image/JoyAI-Image-Edit/ +# Download: modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "joyai_image/JoyAI-Image-Edit/*" --local_dir ./data/diffsynth_example_dataset + +accelerate launch examples/joyai_image/model_training/train.py \ + --dataset_base_path "./data/diffsynth_example_dataset/joyai_image/JoyAI-Image-Edit" \ + --dataset_metadata_path "./data/diffsynth_example_dataset/joyai_image/JoyAI-Image-Edit/metadata.csv" \ + --max_pixels 1048576 \ + --dataset_repeat 1 \ + --model_id_with_origin_paths "jd-opensource/JoyAI-Image-Edit:JoyAI-Image-Und/model*.safetensors,jd-opensource/JoyAI-Image-Edit:vae/Wan2.1_VAE.pth" \ + --learning_rate 1e-5 \ + --num_epochs 2 \ + --remove_prefix_in_ckpt "pipe.dit." \ + --output_path "./models/train/JoyAI-Image-Edit-full-cache" \ + --use_gradient_checkpointing \ + --find_unused_parameters \ + --data_file_keys "image,edit_image" \ + --extra_inputs "edit_image" \ + --task "sft:data_process" + +accelerate launch --config_file examples/joyai_image/model_training/full/accelerate_config_zero3.yaml \ + examples/joyai_image/model_training/train.py \ + --dataset_base_path "./models/train/JoyAI-Image-Edit-full-cache" \ + --max_pixels 1048576 \ + --dataset_repeat 50 \ + --model_id_with_origin_paths "jd-opensource/JoyAI-Image-Edit:transformer/transformer.pth" \ + --learning_rate 1e-5 \ + --num_epochs 2 \ + --remove_prefix_in_ckpt "pipe.dit." \ + --output_path "./models/train/JoyAI-Image-Edit-full" \ + --trainable_models "dit" \ + --use_gradient_checkpointing \ + --find_unused_parameters \ + --data_file_keys "image,edit_image" \ + --extra_inputs "edit_image" \ + --task "sft:train" diff --git a/examples/joyai_image/model_training/full/accelerate_config_zero3.yaml b/examples/joyai_image/model_training/full/accelerate_config_zero3.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e6a8d273346c6e04e7ab97c8aa661914a0819a86 --- /dev/null +++ b/examples/joyai_image/model_training/full/accelerate_config_zero3.yaml @@ -0,0 +1,23 @@ +compute_environment: LOCAL_MACHINE +debug: false +deepspeed_config: + gradient_accumulation_steps: 1 + offload_optimizer_device: none + offload_param_device: none + zero3_init_flag: true + zero3_save_16bit_model: true + zero_stage: 3 +distributed_type: DEEPSPEED +downcast_bf16: 'no' +enable_cpu_affinity: false +machine_rank: 0 +main_training_function: main +mixed_precision: bf16 +num_machines: 1 +num_processes: 8 +rdzv_backend: static +same_network: true +tpu_env: [] +tpu_use_cluster: false +tpu_use_sudo: false +use_cpu: false diff --git a/examples/joyai_image/model_training/lora/JoyAI-Image-Edit.sh b/examples/joyai_image/model_training/lora/JoyAI-Image-Edit.sh new file mode 100644 index 0000000000000000000000000000000000000000..b9a5fd4316302ed016f609975b2c0d5ad1bb6f95 --- /dev/null +++ b/examples/joyai_image/model_training/lora/JoyAI-Image-Edit.sh @@ -0,0 +1,39 @@ +# Dataset: data/diffsynth_example_dataset/joyai_image/JoyAI-Image-Edit/ +# Download: modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "joyai_image/JoyAI-Image-Edit/*" --local_dir ./data/diffsynth_example_dataset + +accelerate launch examples/joyai_image/model_training/train.py \ + --dataset_base_path "./data/diffsynth_example_dataset/joyai_image/JoyAI-Image-Edit" \ + --dataset_metadata_path "./data/diffsynth_example_dataset/joyai_image/JoyAI-Image-Edit/metadata.csv" \ + --max_pixels 1048576 \ + --dataset_repeat 1 \ + --model_id_with_origin_paths "jd-opensource/JoyAI-Image-Edit:JoyAI-Image-Und/model*.safetensors,jd-opensource/JoyAI-Image-Edit:vae/Wan2.1_VAE.pth" \ + --learning_rate 1e-4 \ + --num_epochs 5 \ + --remove_prefix_in_ckpt "pipe.dit." \ + --output_path "./models/train/JoyAI-Image-Edit-split-cache" \ + --lora_base_model "dit" \ + --lora_target_modules "img_attn_qkv,txt_attn_qkv,img_attn_proj,txt_attn_proj" \ + --lora_rank 32 \ + --use_gradient_checkpointing \ + --find_unused_parameters \ + --data_file_keys "image,edit_image" \ + --extra_inputs "edit_image" \ + --task "sft:data_process" + +accelerate launch examples/joyai_image/model_training/train.py \ + --dataset_base_path "./models/train/JoyAI-Image-Edit-split-cache" \ + --max_pixels 1048576 \ + --dataset_repeat 50 \ + --model_id_with_origin_paths "jd-opensource/JoyAI-Image-Edit:transformer/transformer.pth" \ + --learning_rate 1e-4 \ + --num_epochs 5 \ + --remove_prefix_in_ckpt "pipe.dit." \ + --output_path "./models/train/JoyAI-Image-Edit-lora" \ + --lora_base_model "dit" \ + --lora_target_modules "img_attn_qkv,txt_attn_qkv,img_attn_proj,txt_attn_proj" \ + --lora_rank 32 \ + --use_gradient_checkpointing \ + --find_unused_parameters \ + --data_file_keys "image,edit_image" \ + --extra_inputs "edit_image" \ + --task "sft:train" diff --git a/examples/joyai_image/model_training/special/split_training/JoyAI-Image-Edit.sh b/examples/joyai_image/model_training/special/split_training/JoyAI-Image-Edit.sh new file mode 100644 index 0000000000000000000000000000000000000000..3f0c3e46114706a20629b630a891c7689b98f89d --- /dev/null +++ b/examples/joyai_image/model_training/special/split_training/JoyAI-Image-Edit.sh @@ -0,0 +1,42 @@ +modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "joyai_image/JoyAI-Image-Edit/*" --local_dir ./data/diffsynth_example_dataset + +# Stage 1: cache deterministic preprocessing outputs. +accelerate launch examples/joyai_image/model_training/train.py \ + --dataset_base_path ./data/diffsynth_example_dataset/joyai_image/JoyAI-Image-Edit \ + --dataset_metadata_path ./data/diffsynth_example_dataset/joyai_image/JoyAI-Image-Edit/metadata.csv \ + --max_pixels 1048576 \ + --dataset_repeat 1 \ + --model_id_with_origin_paths 'jd-opensource/JoyAI-Image-Edit:transformer/transformer.pth,jd-opensource/JoyAI-Image-Edit:JoyAI-Image-Und/model*.safetensors,jd-opensource/JoyAI-Image-Edit:vae/Wan2.1_VAE.pth' \ + --learning_rate 1e-4 \ + --num_epochs 5 \ + --remove_prefix_in_ckpt pipe.dit. \ + --output_path ./models/train/JoyAI-Image-Edit-split-cache \ + --lora_base_model dit \ + --lora_target_modules img_attn_qkv,txt_attn_qkv,img_attn_proj,txt_attn_proj \ + --lora_rank 32 \ + --use_gradient_checkpointing \ + --find_unused_parameters \ + --data_file_keys image,edit_image \ + --extra_inputs edit_image \ + --task sft:data_process \ + --offload_models jd-opensource/JoyAI-Image-Edit:transformer/transformer.pth + +# Stage 2: train LoRA from the cached dataset. +accelerate launch examples/joyai_image/model_training/train.py \ + --dataset_base_path ./models/train/JoyAI-Image-Edit-split-cache \ + --max_pixels 1048576 \ + --dataset_repeat 50 \ + --model_id_with_origin_paths 'jd-opensource/JoyAI-Image-Edit:transformer/transformer.pth,jd-opensource/JoyAI-Image-Edit:JoyAI-Image-Und/model*.safetensors,jd-opensource/JoyAI-Image-Edit:vae/Wan2.1_VAE.pth' \ + --learning_rate 1e-4 \ + --num_epochs 5 \ + --remove_prefix_in_ckpt pipe.dit. \ + --output_path ./models/train/JoyAI-Image-Edit-split \ + --lora_base_model dit \ + --lora_target_modules img_attn_qkv,txt_attn_qkv,img_attn_proj,txt_attn_proj \ + --lora_rank 32 \ + --use_gradient_checkpointing \ + --find_unused_parameters \ + --data_file_keys image,edit_image \ + --extra_inputs edit_image \ + --task sft:train \ + --offload_models 'jd-opensource/JoyAI-Image-Edit:JoyAI-Image-Und/model*.safetensors,jd-opensource/JoyAI-Image-Edit:vae/Wan2.1_VAE.pth' diff --git a/examples/joyai_image/model_training/special/split_training/validate.py b/examples/joyai_image/model_training/special/split_training/validate.py new file mode 100644 index 0000000000000000000000000000000000000000..4e1b2136b5493ef8774beb88b7465aa01f73c8aa --- /dev/null +++ b/examples/joyai_image/model_training/special/split_training/validate.py @@ -0,0 +1,30 @@ +import torch +from PIL import Image +from diffsynth.pipelines.joyai_image import JoyAIImagePipeline, ModelConfig + +pipe = JoyAIImagePipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="jd-opensource/JoyAI-Image-Edit", origin_file_pattern="transformer/transformer.pth"), + ModelConfig(model_id="jd-opensource/JoyAI-Image-Edit", origin_file_pattern="JoyAI-Image-Und/model*.safetensors"), + ModelConfig(model_id="jd-opensource/JoyAI-Image-Edit", origin_file_pattern="vae/Wan2.1_VAE.pth"), + ], + processor_config=ModelConfig(model_id="jd-opensource/JoyAI-Image-Edit", origin_file_pattern="JoyAI-Image-Und/"), +) + +pipe.load_lora(pipe.dit, './models/train/JoyAI-Image-Edit-split/epoch-4.safetensors') + +prompt = "将裙子改为粉色" +edit_image = Image.open("data/diffsynth_example_dataset/joyai_image/JoyAI-Image-Edit/edit/image1.jpg").convert("RGB") + +image = pipe( + prompt=prompt, + edit_image=edit_image, + height=1024, + width=1024, + seed=0, + num_inference_steps=30, + cfg_scale=5.0, +) +image.save('split_training_JoyAI-Image-Edit.jpg') diff --git a/examples/joyai_image/model_training/train.py b/examples/joyai_image/model_training/train.py new file mode 100644 index 0000000000000000000000000000000000000000..f8c16d5b8128c09f20a12b6e8600486cdfb0a483 --- /dev/null +++ b/examples/joyai_image/model_training/train.py @@ -0,0 +1,150 @@ +import torch, os, argparse, accelerate +from diffsynth.core import UnifiedDataset +from diffsynth.pipelines.joyai_image import JoyAIImagePipeline, ModelConfig +from diffsynth.diffusion import * +from diffsynth.core.data.operators import * +os.environ["TOKENIZERS_PARALLELISM"] = "false" + + +class JoyAIImageTrainingModule(DiffusionTrainingModule): + def __init__( + self, + model_paths=None, model_id_with_origin_paths=None, + processor_path=None, + trainable_models=None, + lora_base_model=None, lora_target_modules="", lora_rank=32, lora_checkpoint=None, + preset_lora_path=None, preset_lora_model=None, + use_gradient_checkpointing=True, + use_gradient_checkpointing_offload=False, + extra_inputs=None, + fp8_models=None, + offload_models=None, + quant_options=None, + resume_from_checkpoint=None, remove_prefix_in_ckpt=None, + device="cpu", + task="sft", + ): + super().__init__() + # Load models + model_configs = self.parse_model_configs(model_paths, model_id_with_origin_paths, fp8_models=fp8_models, offload_models=offload_models, quant_options=quant_options, device=device) + processor_config = ModelConfig(model_id="jd-opensource/JoyAI-Image-Edit", origin_file_pattern="JoyAI-Image-Und/") if processor_path is None else ModelConfig(processor_path) + self.pipe = JoyAIImagePipeline.from_pretrained(torch_dtype=torch.bfloat16, device=device, model_configs=model_configs, processor_config=processor_config) + self.pipe = self.split_pipeline_units(task, self.pipe, trainable_models, lora_base_model) + self.resume_from_checkpoint(resume_from_checkpoint, remove_prefix_in_ckpt) + + # Training mode + self.switch_pipe_to_training_mode( + self.pipe, trainable_models, + lora_base_model, lora_target_modules, lora_rank, lora_checkpoint, + preset_lora_path, preset_lora_model, + task=task, + ) + + # Other configs + self.use_gradient_checkpointing = use_gradient_checkpointing + self.use_gradient_checkpointing_offload = use_gradient_checkpointing_offload + self.extra_inputs = extra_inputs.split(",") if extra_inputs is not None else [] + self.fp8_models = fp8_models + self.task = task + self.task_to_loss = { + "sft:data_process": lambda pipe, *args: args, + "sft": lambda pipe, inputs_shared, inputs_posi, inputs_nega: FlowMatchSFTLoss(pipe, **inputs_shared, **inputs_posi), + "sft:train": lambda pipe, inputs_shared, inputs_posi, inputs_nega: FlowMatchSFTLoss(pipe, **inputs_shared, **inputs_posi), + } + + def get_pipeline_inputs(self, data): + inputs_posi = {"prompt": data["prompt"]} + inputs_nega = {"negative_prompt": ""} + inputs_shared = { + # Assume you are using this pipeline for inference, + # please fill in the input parameters. + "input_image": data["image"], + "height": data["image"].size[1], + "width": data["image"].size[0], + # Please do not modify the following parameters + # unless you clearly know what this will cause. + "cfg_scale": 1, + "rand_device": self.pipe.device, + "use_gradient_checkpointing": self.use_gradient_checkpointing, + "use_gradient_checkpointing_offload": self.use_gradient_checkpointing_offload, + } + inputs_shared = self.parse_extra_inputs(data, self.extra_inputs, inputs_shared) + return inputs_shared, inputs_posi, inputs_nega + + def forward(self, data, inputs=None): + if inputs is None: inputs = self.get_pipeline_inputs(data) + inputs = self.transfer_data_to_device(inputs, self.pipe.device, self.pipe.torch_dtype) + for unit in self.pipe.units: + inputs = self.pipe.unit_runner(unit, self.pipe, *inputs) + loss = self.task_to_loss[self.task](self.pipe, *inputs) + return loss + + +def joyai_image_parser(): + parser = argparse.ArgumentParser(description="JoyAI-Image training.") + parser = add_general_config(parser) + parser = add_image_size_config(parser) + parser.add_argument("--processor_path", type=str, default=None, help="Path to the processor.") + parser.add_argument("--initialize_model_on_cpu", default=False, action="store_true", help="Whether to initialize models on CPU.") + return parser + + +if __name__ == "__main__": + parser = joyai_image_parser() + args = parser.parse_args() + accelerator = accelerate.Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + kwargs_handlers=[accelerate.DistributedDataParallelKwargs(find_unused_parameters=args.find_unused_parameters)], + ) + dataset = UnifiedDataset( + base_path=args.dataset_base_path, + metadata_path=args.dataset_metadata_path, + repeat=args.dataset_repeat, + data_file_keys=args.data_file_keys.split(","), + main_data_operator=UnifiedDataset.default_image_operator( + base_path=args.dataset_base_path, + max_pixels=args.max_pixels, + height=args.height, + width=args.width, + height_division_factor=16, + width_division_factor=16, + ), + ) + model = JoyAIImageTrainingModule( + model_paths=args.model_paths, + model_id_with_origin_paths=args.model_id_with_origin_paths, + processor_path=args.processor_path, + trainable_models=args.trainable_models, + lora_base_model=args.lora_base_model, + lora_target_modules=args.lora_target_modules, + lora_rank=args.lora_rank, + lora_checkpoint=args.lora_checkpoint, + preset_lora_path=args.preset_lora_path, + preset_lora_model=args.preset_lora_model, + use_gradient_checkpointing=args.use_gradient_checkpointing, + use_gradient_checkpointing_offload=args.use_gradient_checkpointing_offload, + extra_inputs=args.extra_inputs, + fp8_models=args.fp8_models, + offload_models=args.offload_models, + quant_options=args.quant_options, + resume_from_checkpoint=args.resume_from_checkpoint, + remove_prefix_in_ckpt=args.remove_prefix_in_ckpt, + task=args.task, + device="cpu" if (args.initialize_model_on_cpu or args.enable_model_cpu_offload) else accelerator.device, + ) + model_logger = ModelLogger( + args.output_path, + remove_prefix_in_ckpt=args.remove_prefix_in_ckpt, + enable_tensorboard_log=args.enable_tensorboard_log, + enable_swanlab_log=args.enable_swanlab_log, + swanlab_project=args.swanlab_project, + enable_wandb_log=args.enable_wandb_log, + wandb_project=args.wandb_project, + enable_csv_log=args.enable_csv_log, + ) + launcher_map = { + "sft:data_process": launch_data_process_task, + "sft": launch_training_task, + "sft:train": launch_training_task, + } + launcher_map[args.task](accelerator, dataset, model, model_logger, args=args) diff --git a/examples/joyai_image/model_training/validate_full/JoyAI-Image-Edit.py b/examples/joyai_image/model_training/validate_full/JoyAI-Image-Edit.py new file mode 100644 index 0000000000000000000000000000000000000000..f245deb3ffb45b32e4d0510191a188d1d999d0a6 --- /dev/null +++ b/examples/joyai_image/model_training/validate_full/JoyAI-Image-Edit.py @@ -0,0 +1,32 @@ +import torch +from PIL import Image +from diffsynth.pipelines.joyai_image import JoyAIImagePipeline, ModelConfig +from diffsynth import load_state_dict + +pipe = JoyAIImagePipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="jd-opensource/JoyAI-Image-Edit", origin_file_pattern="transformer/transformer.pth"), + ModelConfig(model_id="jd-opensource/JoyAI-Image-Edit", origin_file_pattern="JoyAI-Image-Und/model*.safetensors"), + ModelConfig(model_id="jd-opensource/JoyAI-Image-Edit", origin_file_pattern="vae/Wan2.1_VAE.pth"), + ], + processor_config=ModelConfig(model_id="jd-opensource/JoyAI-Image-Edit", origin_file_pattern="JoyAI-Image-Und/"), +) + +state_dict = load_state_dict("models/train/JoyAI-Image-Edit_full/epoch-1.safetensors") +pipe.dit.load_state_dict(state_dict) + +prompt = "将裙子改为粉色" +edit_image = Image.open("data/diffsynth_example_dataset/joyai_image/JoyAI-Image-Edit/edit/image1.jpg").convert("RGB") + +image = pipe( + prompt=prompt, + edit_image=edit_image, + height=1024, + width=1024, + seed=0, + num_inference_steps=50, + cfg_scale=5.0, +) +image.save("image_full.jpg") diff --git a/examples/joyai_image/model_training/validate_lora/JoyAI-Image-Edit.py b/examples/joyai_image/model_training/validate_lora/JoyAI-Image-Edit.py new file mode 100644 index 0000000000000000000000000000000000000000..16f06aa64849d28784edd036dbea53217934e76a --- /dev/null +++ b/examples/joyai_image/model_training/validate_lora/JoyAI-Image-Edit.py @@ -0,0 +1,30 @@ +import torch +from PIL import Image +from diffsynth.pipelines.joyai_image import JoyAIImagePipeline, ModelConfig + +pipe = JoyAIImagePipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="jd-opensource/JoyAI-Image-Edit", origin_file_pattern="transformer/transformer.pth"), + ModelConfig(model_id="jd-opensource/JoyAI-Image-Edit", origin_file_pattern="JoyAI-Image-Und/model*.safetensors"), + ModelConfig(model_id="jd-opensource/JoyAI-Image-Edit", origin_file_pattern="vae/Wan2.1_VAE.pth"), + ], + processor_config=ModelConfig(model_id="jd-opensource/JoyAI-Image-Edit", origin_file_pattern="JoyAI-Image-Und/"), +) + +pipe.load_lora(pipe.dit, "models/train/JoyAI-Image-Edit-lora/epoch-4.safetensors") + +prompt = "将裙子改为粉色" +edit_image = Image.open("data/diffsynth_example_dataset/joyai_image/JoyAI-Image-Edit/edit/image1.jpg").convert("RGB") + +image = pipe( + prompt=prompt, + edit_image=edit_image, + height=1024, + width=1024, + seed=0, + num_inference_steps=30, + cfg_scale=5.0, +) +image.save("image_lora.jpg") diff --git a/examples/krea2/model_inference/Krea-2-Raw.py b/examples/krea2/model_inference/Krea-2-Raw.py new file mode 100644 index 0000000000000000000000000000000000000000..c84c676d5346b408ac23f9cd9f4c47a657e9c670 --- /dev/null +++ b/examples/krea2/model_inference/Krea-2-Raw.py @@ -0,0 +1,17 @@ +from diffsynth.pipelines.krea2 import Krea2Pipeline, ModelConfig +import torch + + +pipe = Krea2Pipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="krea/Krea-2-Raw", origin_file_pattern="raw.safetensors"), + ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern="*.safetensors"), + ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="vae/diffusion_pytorch_model.safetensors"), + ], + tokenizer_config=ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern=""), +) +prompt = "A cat standing on a stone." +image = pipe(prompt, seed=0, num_inference_steps=52, cfg_scale=4.5) +image.save("image.jpg") diff --git a/examples/krea2/model_inference/Krea-2-Turbo.py b/examples/krea2/model_inference/Krea-2-Turbo.py new file mode 100644 index 0000000000000000000000000000000000000000..7e50adf2d7c4782206ea0ba0146d79b3180e9e77 --- /dev/null +++ b/examples/krea2/model_inference/Krea-2-Turbo.py @@ -0,0 +1,22 @@ +from diffsynth.pipelines.krea2 import Krea2Pipeline, ModelConfig +import torch + + +pipe = Krea2Pipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="krea/Krea-2-Turbo", origin_file_pattern="turbo.safetensors"), + ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern="*.safetensors"), + ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="vae/diffusion_pytorch_model.safetensors"), + ], + tokenizer_config=ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern=""), +) +prompt = "Portrait of a woman in a blue dress, underwater, surrounded by colorful bubbles." +image = pipe( + prompt, seed=0, + height=2048, width=2048, + # The following parameters are fixed. + num_inference_steps=8, cfg_scale=1, mu=1.15, +) +image.save("image.jpg") diff --git a/examples/krea2/model_inference_low_vram/Krea-2-Raw.py b/examples/krea2/model_inference_low_vram/Krea-2-Raw.py new file mode 100644 index 0000000000000000000000000000000000000000..a440cb70de8bb2a944852a89bd2002b9e8430b4d --- /dev/null +++ b/examples/krea2/model_inference_low_vram/Krea-2-Raw.py @@ -0,0 +1,27 @@ +from diffsynth.pipelines.krea2 import Krea2Pipeline, ModelConfig +import torch + +vram_config = { + "offload_dtype": "disk", + "offload_device": "disk", + "onload_dtype": torch.float8_e4m3fn, + "onload_device": "cpu", + "preparing_dtype": torch.float8_e4m3fn, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = Krea2Pipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="krea/Krea-2-Raw", origin_file_pattern="raw.safetensors", **vram_config), + ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern="*.safetensors", **vram_config), + ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="vae/diffusion_pytorch_model.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern=""), + vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 1, +) +prompt = "A cat standing on a stone." +image = pipe(prompt, seed=0, num_inference_steps=52, cfg_scale=4.5) +image.save("image.jpg") diff --git a/examples/krea2/model_inference_low_vram/Krea-2-Turbo.py b/examples/krea2/model_inference_low_vram/Krea-2-Turbo.py new file mode 100644 index 0000000000000000000000000000000000000000..3200ec74f2ff64e63ee24754e4376b23a2aa085f --- /dev/null +++ b/examples/krea2/model_inference_low_vram/Krea-2-Turbo.py @@ -0,0 +1,32 @@ +from diffsynth.pipelines.krea2 import Krea2Pipeline, ModelConfig +import torch + +vram_config = { + "offload_dtype": "disk", + "offload_device": "disk", + "onload_dtype": torch.float8_e4m3fn, + "onload_device": "cpu", + "preparing_dtype": torch.float8_e4m3fn, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = Krea2Pipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="krea/Krea-2-Turbo", origin_file_pattern="turbo.safetensors", **vram_config), + ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern="*.safetensors", **vram_config), + ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="vae/diffusion_pytorch_model.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern=""), + vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 1, +) +prompt = "Portrait of a woman in a blue dress, underwater, surrounded by colorful bubbles." +image = pipe( + prompt, seed=0, + height=2048, width=2048, + # The following parameters are fixed. + num_inference_steps=8, cfg_scale=1, mu=1.15, +) +image.save("image.jpg") diff --git a/examples/krea2/model_training/full/Krea-2-Raw.sh b/examples/krea2/model_training/full/Krea-2-Raw.sh new file mode 100644 index 0000000000000000000000000000000000000000..9d3fb4c594aa6ad8e8086fec8bbb730f20dbff1c --- /dev/null +++ b/examples/krea2/model_training/full/Krea-2-Raw.sh @@ -0,0 +1,17 @@ +# Please run `accelerate config` to configure GPU, DeepSpeed, etc. +modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "krea2/Krea-2-Raw/*" --local_dir ./data/diffsynth_example_dataset + +accelerate launch examples/krea2/model_training/train.py \ + --dataset_base_path data/diffsynth_example_dataset/krea2/Krea-2-Raw \ + --dataset_metadata_path data/diffsynth_example_dataset/krea2/Krea-2-Raw/metadata.csv \ + --max_pixels 1048576 \ + --dataset_repeat 50 \ + --model_id_with_origin_paths "krea/Krea-2-Raw:raw.safetensors,Qwen/Qwen3-VL-4B-Instruct:*.safetensors,Qwen/Qwen-Image:vae/diffusion_pytorch_model.safetensors" \ + --tokenizer_path "Qwen/Qwen3-VL-4B-Instruct:" \ + --learning_rate 1e-5 \ + --num_epochs 2 \ + --remove_prefix_in_ckpt "pipe.dit." \ + --output_path "./models/train/Krea-2-Raw_full" \ + --trainable_models "dit" \ + --use_gradient_checkpointing \ + --find_unused_parameters diff --git a/examples/krea2/model_training/full/Krea-2-Turbo.sh b/examples/krea2/model_training/full/Krea-2-Turbo.sh new file mode 100644 index 0000000000000000000000000000000000000000..886c891206501c39fe18e694c83df172525a5382 --- /dev/null +++ b/examples/krea2/model_training/full/Krea-2-Turbo.sh @@ -0,0 +1,17 @@ +# Please run `accelerate config` to configure GPU, DeepSpeed, etc. +modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "krea2/Krea-2-Turbo/*" --local_dir ./data/diffsynth_example_dataset + +accelerate launch examples/krea2/model_training/train.py \ + --dataset_base_path data/diffsynth_example_dataset/krea2/Krea-2-Turbo \ + --dataset_metadata_path data/diffsynth_example_dataset/krea2/Krea-2-Turbo/metadata.csv \ + --max_pixels 1048576 \ + --dataset_repeat 50 \ + --model_id_with_origin_paths "krea/Krea-2-Turbo:turbo.safetensors,Qwen/Qwen3-VL-4B-Instruct:*.safetensors,Qwen/Qwen-Image:vae/diffusion_pytorch_model.safetensors" \ + --tokenizer_path "Qwen/Qwen3-VL-4B-Instruct:" \ + --learning_rate 1e-5 \ + --num_epochs 2 \ + --remove_prefix_in_ckpt "pipe.dit." \ + --output_path "./models/train/Krea-2-Turbo_full" \ + --trainable_models "dit" \ + --use_gradient_checkpointing \ + --find_unused_parameters diff --git a/examples/krea2/model_training/lora/Krea-2-Raw.sh b/examples/krea2/model_training/lora/Krea-2-Raw.sh new file mode 100644 index 0000000000000000000000000000000000000000..7fd8513620610661f17bc8092e08b50b6fd1bd38 --- /dev/null +++ b/examples/krea2/model_training/lora/Krea-2-Raw.sh @@ -0,0 +1,19 @@ +modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "krea2/Krea-2-Raw/*" --local_dir ./data/diffsynth_example_dataset + +accelerate launch examples/krea2/model_training/train.py \ + --dataset_base_path data/diffsynth_example_dataset/krea2/Krea-2-Raw \ + --dataset_metadata_path data/diffsynth_example_dataset/krea2/Krea-2-Raw/metadata.csv \ + --max_pixels 1048576 \ + --dataset_repeat 50 \ + --model_id_with_origin_paths "krea/Krea-2-Raw:raw.safetensors,Qwen/Qwen3-VL-4B-Instruct:*.safetensors,Qwen/Qwen-Image:vae/diffusion_pytorch_model.safetensors" \ + --tokenizer_path "Qwen/Qwen3-VL-4B-Instruct:" \ + --learning_rate 1e-4 \ + --num_epochs 5 \ + --remove_prefix_in_ckpt "pipe.dit." \ + --output_path "./models/train/Krea-2-Raw_lora" \ + --lora_base_model "dit" \ + --lora_target_modules "wq,wk,wv,gate,wo,gate,up,down,first,tmlp.0,tmlp.2,projector,txtmlp.1,txtmlp.3,last.linear,tproj.1" \ + --lora_rank 32 \ + --use_gradient_checkpointing \ + --find_unused_parameters \ + --align_to_opensource_format diff --git a/examples/krea2/model_training/lora/Krea-2-Turbo.sh b/examples/krea2/model_training/lora/Krea-2-Turbo.sh new file mode 100644 index 0000000000000000000000000000000000000000..049411b6a23f9112c47be43d01a2ce903a665737 --- /dev/null +++ b/examples/krea2/model_training/lora/Krea-2-Turbo.sh @@ -0,0 +1,21 @@ +# We do not recommend training LoRA on Krea-2-Turbo. + +modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "krea2/Krea-2-Turbo/*" --local_dir ./data/diffsynth_example_dataset + +accelerate launch examples/krea2/model_training/train.py \ + --dataset_base_path data/diffsynth_example_dataset/krea2/Krea-2-Turbo \ + --dataset_metadata_path data/diffsynth_example_dataset/krea2/Krea-2-Turbo/metadata.csv \ + --max_pixels 1048576 \ + --dataset_repeat 50 \ + --model_id_with_origin_paths "krea/Krea-2-Turbo:turbo.safetensors,Qwen/Qwen3-VL-4B-Instruct:*.safetensors,Qwen/Qwen-Image:vae/diffusion_pytorch_model.safetensors" \ + --tokenizer_path "Qwen/Qwen3-VL-4B-Instruct:" \ + --learning_rate 1e-4 \ + --num_epochs 5 \ + --remove_prefix_in_ckpt "pipe.dit." \ + --output_path "./models/train/Krea-2-Turbo_lora" \ + --lora_base_model "dit" \ + --lora_target_modules "wq,wk,wv,gate,wo,gate,up,down,first,tmlp.0,tmlp.2,projector,txtmlp.1,txtmlp.3,last.linear,tproj.1" \ + --lora_rank 32 \ + --use_gradient_checkpointing \ + --find_unused_parameters \ + --align_to_opensource_format diff --git a/examples/krea2/model_training/special/split_training/Krea-2-Raw.sh b/examples/krea2/model_training/special/split_training/Krea-2-Raw.sh new file mode 100644 index 0000000000000000000000000000000000000000..5218e74cb80ccaa33ae9589d63a8851707ccd7df --- /dev/null +++ b/examples/krea2/model_training/special/split_training/Krea-2-Raw.sh @@ -0,0 +1,42 @@ +modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "krea2/Krea-2-Raw/*" --local_dir ./data/diffsynth_example_dataset + +# Stage 1: cache deterministic preprocessing outputs. +accelerate launch examples/krea2/model_training/train.py \ + --dataset_base_path data/diffsynth_example_dataset/krea2/Krea-2-Raw \ + --dataset_metadata_path data/diffsynth_example_dataset/krea2/Krea-2-Raw/metadata.csv \ + --max_pixels 1048576 \ + --dataset_repeat 1 \ + --model_id_with_origin_paths 'krea/Krea-2-Raw:raw.safetensors,Qwen/Qwen3-VL-4B-Instruct:*.safetensors,Qwen/Qwen-Image:vae/diffusion_pytorch_model.safetensors' \ + --tokenizer_path Qwen/Qwen3-VL-4B-Instruct: \ + --learning_rate 1e-4 \ + --num_epochs 5 \ + --remove_prefix_in_ckpt pipe.dit. \ + --output_path ./models/train/Krea-2-Raw_split_cache \ + --lora_base_model dit \ + --lora_target_modules wq,wk,wv,gate,wo,gate,up,down,first,tmlp.0,tmlp.2,projector,txtmlp.1,txtmlp.3,last.linear,tproj.1 \ + --lora_rank 32 \ + --use_gradient_checkpointing \ + --find_unused_parameters \ + --align_to_opensource_format \ + --offload_models krea/Krea-2-Raw:raw.safetensors \ + --task sft:data_process + +# Stage 2: train LoRA from the cached dataset. +accelerate launch examples/krea2/model_training/train.py \ + --dataset_base_path ./models/train/Krea-2-Raw_split_cache \ + --max_pixels 1048576 \ + --dataset_repeat 50 \ + --model_id_with_origin_paths 'krea/Krea-2-Raw:raw.safetensors,Qwen/Qwen3-VL-4B-Instruct:*.safetensors,Qwen/Qwen-Image:vae/diffusion_pytorch_model.safetensors' \ + --tokenizer_path Qwen/Qwen3-VL-4B-Instruct: \ + --learning_rate 1e-4 \ + --num_epochs 5 \ + --remove_prefix_in_ckpt pipe.dit. \ + --output_path ./models/train/Krea-2-Raw_split \ + --lora_base_model dit \ + --lora_target_modules wq,wk,wv,gate,wo,gate,up,down,first,tmlp.0,tmlp.2,projector,txtmlp.1,txtmlp.3,last.linear,tproj.1 \ + --lora_rank 32 \ + --use_gradient_checkpointing \ + --find_unused_parameters \ + --align_to_opensource_format \ + --offload_models 'Qwen/Qwen3-VL-4B-Instruct:*.safetensors,Qwen/Qwen-Image:vae/diffusion_pytorch_model.safetensors' \ + --task sft:train diff --git a/examples/krea2/model_training/special/split_training/validate.py b/examples/krea2/model_training/special/split_training/validate.py new file mode 100644 index 0000000000000000000000000000000000000000..28e379060ccc5919af88fc5816ed394b240c6f16 --- /dev/null +++ b/examples/krea2/model_training/special/split_training/validate.py @@ -0,0 +1,19 @@ +from diffsynth.pipelines.krea2 import Krea2Pipeline, ModelConfig +import torch + + +pipe = Krea2Pipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + # For LoRA models trained on Krea-2-Raw, we recommend using them on Krea-2-Turbo. + ModelConfig(model_id="krea/Krea-2-Raw", origin_file_pattern="raw.safetensors"), + ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern="*.safetensors"), + ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="vae/diffusion_pytorch_model.safetensors"), + ], + tokenizer_config=ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern=""), +) +pipe.load_lora(pipe.dit, './models/train/Krea-2-Raw_split/epoch-4.safetensors') +prompt = "A dog" +image = pipe(prompt, seed=0, num_inference_steps=52, cfg_scale=4.5) +image.save('split_training_Krea-2-Raw.jpg') diff --git a/examples/krea2/model_training/train.py b/examples/krea2/model_training/train.py new file mode 100644 index 0000000000000000000000000000000000000000..92548dad74268504912e74efab958cc181c29ff4 --- /dev/null +++ b/examples/krea2/model_training/train.py @@ -0,0 +1,151 @@ +import torch, os, argparse, accelerate +from diffsynth.core import UnifiedDataset +from diffsynth.pipelines.krea2 import Krea2Pipeline, ModelConfig +from diffsynth.diffusion import * +from diffsynth.utils.lora.krea2 import Krea2LoRAConverter +os.environ["TOKENIZERS_PARALLELISM"] = "false" + + +class Krea2ImageTrainingModule(DiffusionTrainingModule): + def __init__( + self, + model_paths=None, model_id_with_origin_paths=None, + tokenizer_path=None, + trainable_models=None, + lora_base_model=None, lora_target_modules="", lora_rank=32, lora_checkpoint=None, + preset_lora_path=None, preset_lora_model=None, + use_gradient_checkpointing=True, + use_gradient_checkpointing_offload=False, + extra_inputs=None, + fp8_models=None, + offload_models=None, + quant_options=None, + template_model_id_or_path=None, + resume_from_checkpoint=None, remove_prefix_in_ckpt=None, + enable_lora_hot_loading=False, + device="cpu", + task="sft", + ): + super().__init__() + model_configs = self.parse_model_configs(model_paths, model_id_with_origin_paths, fp8_models=fp8_models, offload_models=offload_models, quant_options=quant_options, device=device) + tokenizer_config = self.parse_path_or_model_id(tokenizer_path, default_value=ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern="")) + self.pipe = Krea2Pipeline.from_pretrained(torch_dtype=torch.bfloat16, device=device, model_configs=model_configs, tokenizer_config=tokenizer_config) + self.pipe = self.load_training_template_model(self.pipe, template_model_id_or_path, use_gradient_checkpointing, use_gradient_checkpointing_offload) + self.pipe = self.split_pipeline_units(task, self.pipe, trainable_models, lora_base_model, remove_unnecessary_params=True) + self.resume_from_checkpoint(resume_from_checkpoint, remove_prefix_in_ckpt) + if enable_lora_hot_loading: self.pipe.dit = self.pipe.enable_lora_hot_loading(self.pipe.dit) + + self.switch_pipe_to_training_mode( + self.pipe, trainable_models, + lora_base_model, lora_target_modules, lora_rank, lora_checkpoint, + preset_lora_path, preset_lora_model, + task=task, + ) + + self.use_gradient_checkpointing = use_gradient_checkpointing + self.use_gradient_checkpointing_offload = use_gradient_checkpointing_offload + self.extra_inputs = extra_inputs.split(",") if extra_inputs is not None else [] + self.fp8_models = fp8_models + self.task = task + self.task_to_loss = { + "sft:data_process": lambda pipe, *args: args, + "sft": lambda pipe, inputs_shared, inputs_posi, inputs_nega: FlowMatchSFTLoss(pipe, **inputs_shared, **inputs_posi), + "sft:train": lambda pipe, inputs_shared, inputs_posi, inputs_nega: FlowMatchSFTLoss(pipe, **inputs_shared, **inputs_posi), + } + + def get_pipeline_inputs(self, data): + inputs_posi = {"prompt": data["prompt"]} + inputs_nega = {"negative_prompt": ""} + inputs_shared = { + "input_image": data["image"], + "height": data["image"].size[1], + "width": data["image"].size[0], + "cfg_scale": 1, + "rand_device": self.pipe.device, + "use_gradient_checkpointing": self.use_gradient_checkpointing, + "use_gradient_checkpointing_offload": self.use_gradient_checkpointing_offload, + } + inputs_shared = self.parse_extra_inputs(data, self.extra_inputs, inputs_shared) + return inputs_shared, inputs_posi, inputs_nega + + def forward(self, data, inputs=None): + if inputs is None: inputs = self.get_pipeline_inputs(data) + inputs = self.transfer_data_to_device(inputs, self.pipe.device, self.pipe.torch_dtype) + for unit in self.pipe.units: + inputs = self.pipe.unit_runner(unit, self.pipe, *inputs) + loss = self.task_to_loss[self.task](self.pipe, *inputs) + return loss + + +def krea2_parser(): + parser = argparse.ArgumentParser(description="Simple example of a training script.") + parser = add_general_config(parser) + parser = add_image_size_config(parser) + parser.add_argument("--tokenizer_path", type=str, default=None, help="Path to tokenizer.") + parser.add_argument("--initialize_model_on_cpu", default=False, action="store_true", help="Whether to initialize models on CPU.") + parser.add_argument("--align_to_opensource_format", default=False, action="store_true", help="Whether to align the lora format to opensource format. Example: https://modelscope.cn/models/krea/Krea-2-LoRA-softwatercolor") + return parser + + +if __name__ == "__main__": + parser = krea2_parser() + args = parser.parse_args() + accelerator = accelerate.Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + kwargs_handlers=[accelerate.DistributedDataParallelKwargs(find_unused_parameters=args.find_unused_parameters)], + ) + dataset = UnifiedDataset( + base_path=args.dataset_base_path, + metadata_path=args.dataset_metadata_path, + repeat=args.dataset_repeat, + data_file_keys=args.data_file_keys.split(","), + main_data_operator=UnifiedDataset.default_image_operator( + base_path=args.dataset_base_path, + max_pixels=args.max_pixels, + height=args.height, + width=args.width, + height_division_factor=16, + width_division_factor=16, + ) + ) + model = Krea2ImageTrainingModule( + model_paths=args.model_paths, + model_id_with_origin_paths=args.model_id_with_origin_paths, + tokenizer_path=args.tokenizer_path, + trainable_models=args.trainable_models, + lora_base_model=args.lora_base_model, + lora_target_modules=args.lora_target_modules, + lora_rank=args.lora_rank, + lora_checkpoint=args.lora_checkpoint, + preset_lora_path=args.preset_lora_path, + preset_lora_model=args.preset_lora_model, + use_gradient_checkpointing=args.use_gradient_checkpointing, + use_gradient_checkpointing_offload=args.use_gradient_checkpointing_offload, + extra_inputs=args.extra_inputs, + fp8_models=args.fp8_models, + offload_models=args.offload_models, + quant_options=args.quant_options, + template_model_id_or_path=args.template_model_id_or_path, + resume_from_checkpoint=args.resume_from_checkpoint, + remove_prefix_in_ckpt=args.remove_prefix_in_ckpt, + enable_lora_hot_loading=args.enable_lora_hot_loading, + task=args.task, + device="cpu" if (args.initialize_model_on_cpu or args.enable_model_cpu_offload) else accelerator.device, + ) + model_logger = ModelLogger( + args.output_path, + remove_prefix_in_ckpt=args.remove_prefix_in_ckpt, + state_dict_converter=Krea2LoRAConverter.align_to_opensource_format if args.align_to_opensource_format else lambda x:x, + enable_tensorboard_log=args.enable_tensorboard_log, + enable_swanlab_log=args.enable_swanlab_log, + swanlab_project=args.swanlab_project, + enable_wandb_log=args.enable_wandb_log, + wandb_project=args.wandb_project, + enable_csv_log=args.enable_csv_log, + ) + launcher_map = { + "sft:data_process": launch_data_process_task, + "sft": launch_training_task, + "sft:train": launch_training_task, + } + launcher_map[args.task](accelerator, dataset, model, model_logger, args=args) diff --git a/examples/krea2/model_training/validate_full/Krea-2-Raw.py b/examples/krea2/model_training/validate_full/Krea-2-Raw.py new file mode 100644 index 0000000000000000000000000000000000000000..27f92922e866fbf0e176548ca884b4bc74c419e4 --- /dev/null +++ b/examples/krea2/model_training/validate_full/Krea-2-Raw.py @@ -0,0 +1,19 @@ +from diffsynth.pipelines.krea2 import Krea2Pipeline, ModelConfig +import torch +from diffsynth import load_state_dict + + +pipe = Krea2Pipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="krea/Krea-2-Raw", origin_file_pattern="raw.safetensors"), + ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern="*.safetensors"), + ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="vae/diffusion_pytorch_model.safetensors"), + ], + tokenizer_config=ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern=""), +) +pipe.dit.load_state_dict(load_state_dict("models/train/Krea-2-Raw_full/epoch-1.safetensors")) +prompt = "A dog" +image = pipe(prompt, seed=0, num_inference_steps=52, cfg_scale=4.5) +image.save("image.jpg") diff --git a/examples/krea2/model_training/validate_full/Krea-2-Turbo.py b/examples/krea2/model_training/validate_full/Krea-2-Turbo.py new file mode 100644 index 0000000000000000000000000000000000000000..fe438bbc8357d6590347556757fc904bcbad64b8 --- /dev/null +++ b/examples/krea2/model_training/validate_full/Krea-2-Turbo.py @@ -0,0 +1,19 @@ +from diffsynth.pipelines.krea2 import Krea2Pipeline, ModelConfig +import torch +from diffsynth import load_state_dict + + +pipe = Krea2Pipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="krea/Krea-2-Turbo", origin_file_pattern="turbo.safetensors"), + ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern="*.safetensors"), + ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="vae/diffusion_pytorch_model.safetensors"), + ], + tokenizer_config=ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern=""), +) +pipe.dit.load_state_dict(load_state_dict("models/train/Krea-2-Turbo_full/epoch-1.safetensors")) +prompt = "A dog" +image = pipe(prompt, seed=0, num_inference_steps=52, cfg_scale=4.5) +image.save("image.jpg") diff --git a/examples/krea2/model_training/validate_lora/Krea-2-Raw.py b/examples/krea2/model_training/validate_lora/Krea-2-Raw.py new file mode 100644 index 0000000000000000000000000000000000000000..b97e30c29f8f9d30272a271ccfd614f3b940e951 --- /dev/null +++ b/examples/krea2/model_training/validate_lora/Krea-2-Raw.py @@ -0,0 +1,19 @@ +from diffsynth.pipelines.krea2 import Krea2Pipeline, ModelConfig +import torch + + +pipe = Krea2Pipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + # For LoRA models trained on Krea-2-Raw, we recommend using them on Krea-2-Turbo. + ModelConfig(model_id="krea/Krea-2-Raw", origin_file_pattern="raw.safetensors"), + ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern="*.safetensors"), + ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="vae/diffusion_pytorch_model.safetensors"), + ], + tokenizer_config=ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern=""), +) +pipe.load_lora(pipe.dit, "models/train/Krea-2-Raw_lora/epoch-4.safetensors") +prompt = "A dog" +image = pipe(prompt, seed=0, num_inference_steps=52, cfg_scale=4.5) +image.save("image.jpg") diff --git a/examples/krea2/model_training/validate_lora/Krea-2-Turbo.py b/examples/krea2/model_training/validate_lora/Krea-2-Turbo.py new file mode 100644 index 0000000000000000000000000000000000000000..f06ef857be029bec8a06d0b916658987fa6be918 --- /dev/null +++ b/examples/krea2/model_training/validate_lora/Krea-2-Turbo.py @@ -0,0 +1,18 @@ +from diffsynth.pipelines.krea2 import Krea2Pipeline, ModelConfig +import torch + + +pipe = Krea2Pipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="krea/Krea-2-Turbo", origin_file_pattern="turbo.safetensors"), + ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern="*.safetensors"), + ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="vae/diffusion_pytorch_model.safetensors"), + ], + tokenizer_config=ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern=""), +) +pipe.load_lora(pipe.dit, "models/train/Krea-2-Turbo_lora/epoch-4.safetensors") +prompt = "A dog" +image = pipe(prompt, seed=0, num_inference_steps=52, cfg_scale=4.5) +image.save("image.jpg") diff --git a/examples/lingbot_video/README.md b/examples/lingbot_video/README.md new file mode 100644 index 0000000000000000000000000000000000000000..cdf640d38b6e7173700725d1545b4a8eb594833e --- /dev/null +++ b/examples/lingbot_video/README.md @@ -0,0 +1,3 @@ +English Document: https://diffsynth-studio-doc.readthedocs.io/en/latest/Model_Details/LingBot-Video.html + +中文文档：https://diffsynth-studio-doc.readthedocs.io/zh-cn/latest/Model_Details/LingBot-Video.html diff --git a/examples/lingbot_video/model_inference/lingbot-video-dense-1.3b_t2i.py b/examples/lingbot_video/model_inference/lingbot-video-dense-1.3b_t2i.py new file mode 100644 index 0000000000000000000000000000000000000000..e1779f91ac625edc042671ed93f40c567ffa9800 --- /dev/null +++ b/examples/lingbot_video/model_inference/lingbot-video-dense-1.3b_t2i.py @@ -0,0 +1,32 @@ +import json +import torch +from diffsynth.pipelines.lingbot_video import LingBotVideoPipeline, ModelConfig +from modelscope import dataset_snapshot_download + +pipe = LingBotVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="Robbyant/lingbot-video-dense-1.3b", origin_file_pattern="transformer/diffusion_pytorch_model.safetensors"), + ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern="*.safetensors"), + ModelConfig(model_id="Robbyant/lingbot-video-dense-1.3b", origin_file_pattern="vae/diffusion_pytorch_model.safetensors"), + ], + processor_config=ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern=""), +) + +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/diffsynth_example_dataset", + local_dir="data/diffsynth_example_dataset", + allow_file_pattern="lingbot_video/lingbot-video-dense-1.3b_t2i/*", +) +with open("data/diffsynth_example_dataset/lingbot_video/lingbot-video-dense-1.3b_t2i/t2i_example.json", "r", encoding="utf-8") as f: + caption = json.load(f) + +frames = pipe( + prompt=caption, + negative_prompt=pipe.default_negative_prompt_image, + height=480, width=832, num_frames=1, + num_inference_steps=40, cfg_scale=3.0, + seed=0, +) +frames[0].save("image_lingbot-video-dense-1.3b_t2i.png") diff --git a/examples/lingbot_video/model_inference/lingbot-video-dense-1.3b_t2v.py b/examples/lingbot_video/model_inference/lingbot-video-dense-1.3b_t2v.py new file mode 100644 index 0000000000000000000000000000000000000000..c6f63dc305e69b5703ca9246403baa6ca1e2ad0d --- /dev/null +++ b/examples/lingbot_video/model_inference/lingbot-video-dense-1.3b_t2v.py @@ -0,0 +1,68 @@ +import torch +import json +from diffsynth.utils.data import save_video, VideoData +from diffsynth.pipelines.lingbot_video import LingBotVideoPipeline, ModelConfig +from modelscope import dataset_snapshot_download + +pipe = LingBotVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="Robbyant/lingbot-video-dense-1.3b", origin_file_pattern="transformer/diffusion_pytorch_model.safetensors"), + ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern="*.safetensors"), + ModelConfig(model_id="Robbyant/lingbot-video-dense-1.3b", origin_file_pattern="vae/diffusion_pytorch_model.safetensors"), + ], + processor_config=ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern=""), +) + +# --- Text-to-video ------------------------------------------------------------------- +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/diffsynth_example_dataset", + local_dir="data/diffsynth_example_dataset", + allow_file_pattern="lingbot_video/lingbot-video-dense-1.3b_t2v/*", +) +# LingBot-Video is trained on structured-JSON captions, not free-form prose. This example +# runs on a released in-distribution caption; see the bottom for turning a brief idea into +# such a caption with the two-stage prompt rewriter. +with open("data/diffsynth_example_dataset/lingbot_video/lingbot-video-dense-1.3b_t2v/t2v_example_1.json", "r", encoding="utf-8") as f: + caption = json.load(f) + +video = pipe( + prompt=caption, + negative_prompt=pipe.default_negative_prompt, + height=480, width=832, num_frames=81, + num_inference_steps=40, cfg_scale=3.0, + seed=0, +) +save_video(video, "video_lingbot-video-dense-1.3b_t2v.mp4", fps=15, quality=10) + +# --- Video-to-video ------------------------------------------------------------------ +# denoising_strength < 1 keeps part of the input structure. +input_video = VideoData("video_lingbot-video-dense-1.3b_t2v.mp4", height=480, width=832) +video = pipe( + prompt=caption, + negative_prompt=pipe.default_negative_prompt, + input_video=input_video, denoising_strength=0.7, + height=480, width=832, num_frames=81, + num_inference_steps=40, cfg_scale=3.0, + seed=1, +) +save_video(video, "video_lingbot-video-dense-1.3b_v2v.mp4", fps=15, quality=10) + +# --- Optional: rewrite a brief idea into a structured caption ------------------------ +# The two-stage rewriter (model_training/scripts/prompt_rewriter.py) is a separate VLM + +# stage-2 LoRA adapter (NOT the DiT) and is not downloaded automatically. Fetch both +# weights and point the env vars at them: +# modelscope download --model Qwen/Qwen3.6-27B --local_dir ./models/Qwen/Qwen3.6-27B +# modelscope download --model Robbyant/lingbot-video-rewriter-lora --local_dir ./models/Robbyant/lingbot-video-rewriter-lora +# export REWRITER_BASE_MODEL=./models/Qwen/Qwen3.6-27B +# export REWRITER_ADAPTER=./models/Robbyant/lingbot-video-rewriter-lora +# +# from examples.lingbot_video.model_training.scripts.prompt_rewriter import rewrite_prompt +# caption = rewrite_prompt( +# "A playful puppy runs across a lush green meadow, chasing a red ball. " +# "Dynamic side-tracking camera.", +# mode="t2v", duration=5, +# ) +# video = pipe(prompt=caption, height=480, width=832, num_frames=81, cfg_scale=3.0, seed=0) +# save_video(video, "video_lingbot-video-dense-1.3b_rewrite.mp4", fps=15, quality=10) diff --git a/examples/lingbot_video/model_inference/lingbot-video-dense-1.3b_ti2v.py b/examples/lingbot_video/model_inference/lingbot-video-dense-1.3b_ti2v.py new file mode 100644 index 0000000000000000000000000000000000000000..cb5be5def2d639c0d177627d4e85fe14ac2077b6 --- /dev/null +++ b/examples/lingbot_video/model_inference/lingbot-video-dense-1.3b_ti2v.py @@ -0,0 +1,39 @@ +import os +import json +import torch +from PIL import Image +from diffsynth.utils.data import save_video +from diffsynth.pipelines.lingbot_video import LingBotVideoPipeline, ModelConfig +from modelscope import dataset_snapshot_download + +pipe = LingBotVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="Robbyant/lingbot-video-dense-1.3b", origin_file_pattern="transformer/diffusion_pytorch_model.safetensors"), + ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern="*.safetensors"), + ModelConfig(model_id="Robbyant/lingbot-video-dense-1.3b", origin_file_pattern="vae/diffusion_pytorch_model.safetensors"), + ], + processor_config=ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern=""), +) + +# The condition first frame and its paired caption ship in the example dataset. +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/diffsynth_example_dataset", + local_dir="data/diffsynth_example_dataset", + allow_file_pattern="lingbot_video/lingbot-video-dense-1.3b_ti2v/*", +) +base = "data/diffsynth_example_dataset/lingbot_video/lingbot-video-dense-1.3b_ti2v" +with open(os.path.join(base, "ti2v_example.json"), "r", encoding="utf-8") as f: + caption = json.load(f) +input_image = Image.open(os.path.join(base, "ti2v_first_frame.png")).convert("RGB") + +video = pipe( + prompt=caption, + negative_prompt=pipe.default_negative_prompt, + input_image=input_image, + height=480, width=832, num_frames=81, + num_inference_steps=40, cfg_scale=3.0, + seed=0, +) +save_video(video, "video_lingbot-video-dense-1.3b_ti2v.mp4", fps=15, quality=10) diff --git a/examples/lingbot_video/model_inference/lingbot-video-moe-30b-a3b_t2i.py b/examples/lingbot_video/model_inference/lingbot-video-moe-30b-a3b_t2i.py new file mode 100644 index 0000000000000000000000000000000000000000..1264f521526bbe65212742f5a920087c06eca713 --- /dev/null +++ b/examples/lingbot_video/model_inference/lingbot-video-moe-30b-a3b_t2i.py @@ -0,0 +1,32 @@ +import json +import torch +from diffsynth.pipelines.lingbot_video import LingBotVideoPipeline, ModelConfig +from modelscope import dataset_snapshot_download + +pipe = LingBotVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="transformer/diffusion_pytorch_model*.safetensors"), + ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern="*.safetensors"), + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="vae/diffusion_pytorch_model.safetensors"), + ], + processor_config=ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern=""), +) + +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/diffsynth_example_dataset", + local_dir="data/diffsynth_example_dataset", + allow_file_pattern="lingbot_video/lingbot-video-moe-30b-a3b_t2i/*", +) +with open("data/diffsynth_example_dataset/lingbot_video/lingbot-video-moe-30b-a3b_t2i/t2i_example.json", "r", encoding="utf-8") as f: + caption = json.load(f) + +frames = pipe( + prompt=caption, + negative_prompt=pipe.default_negative_prompt_image, + height=480, width=832, num_frames=1, + num_inference_steps=40, cfg_scale=3.0, + seed=0, +) +frames[0].save("image_lingbot-video-moe-30b-a3b_t2i.png") diff --git a/examples/lingbot_video/model_inference/lingbot-video-moe-30b-a3b_t2v.py b/examples/lingbot_video/model_inference/lingbot-video-moe-30b-a3b_t2v.py new file mode 100644 index 0000000000000000000000000000000000000000..c2d20f90b3966a2b9c6bc8ca93d9533a9f237508 --- /dev/null +++ b/examples/lingbot_video/model_inference/lingbot-video-moe-30b-a3b_t2v.py @@ -0,0 +1,50 @@ +import torch +import json +from diffsynth.utils.data import save_video, VideoData +from diffsynth.pipelines.lingbot_video import LingBotVideoPipeline, ModelConfig +from modelscope import dataset_snapshot_download + +pipe = LingBotVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="transformer/diffusion_pytorch_model*.safetensors"), + ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern="*.safetensors"), + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="vae/diffusion_pytorch_model.safetensors"), + ], + processor_config=ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern=""), +) + +# --- Text-to-video ------------------------------------------------------------------- +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/diffsynth_example_dataset", + local_dir="data/diffsynth_example_dataset", + allow_file_pattern="lingbot_video/lingbot-video-moe-30b-a3b_t2v/*", +) +# LingBot-Video is trained on structured-JSON captions, not free-form prose. This example +# runs on a released in-distribution caption; see the bottom for turning a brief idea into +# such a caption with the two-stage prompt rewriter. +with open("data/diffsynth_example_dataset/lingbot_video/lingbot-video-moe-30b-a3b_t2v/t2v_example_1.json", "r", encoding="utf-8") as f: + caption = json.load(f) + +video = pipe( + prompt=caption, + negative_prompt=pipe.default_negative_prompt, + height=480, width=832, num_frames=81, + num_inference_steps=40, cfg_scale=3.0, + seed=0, +) +save_video(video, "video_lingbot-video-moe-30b-a3b_t2v.mp4", fps=15, quality=10) + +# --- Video-to-video ------------------------------------------------------------------ +# denoising_strength < 1 keeps part of the input structure. +input_video = VideoData("video_lingbot-video-moe-30b-a3b_t2v.mp4", height=480, width=832) +video = pipe( + prompt=caption, + negative_prompt=pipe.default_negative_prompt, + input_video=input_video, denoising_strength=0.7, + height=480, width=832, num_frames=81, + num_inference_steps=40, cfg_scale=3.0, + seed=1, +) +save_video(video, "video_lingbot-video-moe-30b-a3b_v2v.mp4", fps=15, quality=10) diff --git a/examples/lingbot_video/model_inference/lingbot-video-moe-30b-a3b_t2v_refiner.py b/examples/lingbot_video/model_inference/lingbot-video-moe-30b-a3b_t2v_refiner.py new file mode 100644 index 0000000000000000000000000000000000000000..8df03eb5b46c6bf29c78830a58cff6bf00c31fdd --- /dev/null +++ b/examples/lingbot_video/model_inference/lingbot-video-moe-30b-a3b_t2v_refiner.py @@ -0,0 +1,49 @@ +import torch +import json +from diffsynth.utils.data import save_video, VideoData +from diffsynth.pipelines.lingbot_video import LingBotVideoPipeline, ModelConfig +from modelscope import dataset_snapshot_download + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cpu", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} + +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/diffsynth_example_dataset", + local_dir="data/diffsynth_example_dataset", + allow_file_pattern="lingbot_video/lingbot-video-dense-1.3b_t2v/*", +) +base = "data/diffsynth_example_dataset/lingbot_video/lingbot-video-dense-1.3b_t2v" +with open(f"{base}/t2v_example_1.json", "r", encoding="utf-8") as f: + caption = json.load(f) + +pipe = LingBotVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="refiner/diffusion_pytorch_model*.safetensors", **vram_config), + ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern="*.safetensors", **vram_config), + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="vae/diffusion_pytorch_model.safetensors", **vram_config), + ], + processor_config=ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern=""), + vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 5, +) + +input_video = VideoData(f"{base}/video_1.mp4", height=1088, width=1920) +video = pipe( + prompt=caption, + negative_prompt=pipe.default_negative_prompt, + input_video=input_video, + height=1088, width=1920, num_frames=81, + num_inference_steps=8, cfg_scale=3.0, + t_thresh=0.85, sigma_tail_steps=2, + seed=0, +) +save_video(video, "video_lingbot-video-moe-30b-a3b_t2v_refined.mp4", fps=15, quality=10) diff --git a/examples/lingbot_video/model_inference/lingbot-video-moe-30b-a3b_ti2v.py b/examples/lingbot_video/model_inference/lingbot-video-moe-30b-a3b_ti2v.py new file mode 100644 index 0000000000000000000000000000000000000000..931630ad0603c6802eceb4fa7b4f2865fd30f803 --- /dev/null +++ b/examples/lingbot_video/model_inference/lingbot-video-moe-30b-a3b_ti2v.py @@ -0,0 +1,39 @@ +import os +import json +import torch +from PIL import Image +from diffsynth.utils.data import save_video +from diffsynth.pipelines.lingbot_video import LingBotVideoPipeline, ModelConfig +from modelscope import dataset_snapshot_download + +pipe = LingBotVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="transformer/diffusion_pytorch_model*.safetensors"), + ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern="*.safetensors"), + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="vae/diffusion_pytorch_model.safetensors"), + ], + processor_config=ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern=""), +) + +# The condition first frame and its paired caption ship in the example dataset. +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/diffsynth_example_dataset", + local_dir="data/diffsynth_example_dataset", + allow_file_pattern="lingbot_video/lingbot-video-moe-30b-a3b_ti2v/*", +) +base = "data/diffsynth_example_dataset/lingbot_video/lingbot-video-moe-30b-a3b_ti2v" +with open(os.path.join(base, "ti2v_example.json"), "r", encoding="utf-8") as f: + caption = json.load(f) +input_image = Image.open(os.path.join(base, "ti2v_first_frame.png")).convert("RGB") + +video = pipe( + prompt=caption, + negative_prompt=pipe.default_negative_prompt, + input_image=input_image, + height=480, width=832, num_frames=81, + num_inference_steps=40, cfg_scale=3.0, + seed=0, +) +save_video(video, "video_lingbot-video-moe-30b-a3b_ti2v.mp4", fps=15, quality=10) diff --git a/examples/lingbot_video/model_inference/lingbot-video-moe-30b-a3b_ti2v_refiner.py b/examples/lingbot_video/model_inference/lingbot-video-moe-30b-a3b_ti2v_refiner.py new file mode 100644 index 0000000000000000000000000000000000000000..dc1859bc06458627ff4c74baf9b07cae381fb77b --- /dev/null +++ b/examples/lingbot_video/model_inference/lingbot-video-moe-30b-a3b_ti2v_refiner.py @@ -0,0 +1,53 @@ +import os +import json +import torch +from PIL import Image +from diffsynth.utils.data import save_video, VideoData +from diffsynth.pipelines.lingbot_video import LingBotVideoPipeline, ModelConfig +from modelscope import dataset_snapshot_download + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cpu", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} + +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/diffsynth_example_dataset", + local_dir="data/diffsynth_example_dataset", + allow_file_pattern="lingbot_video/lingbot-video-dense-1.3b_ti2v/*", +) +base = "data/diffsynth_example_dataset/lingbot_video/lingbot-video-dense-1.3b_ti2v" +with open(os.path.join(base, "ti2v_example.json"), "r", encoding="utf-8") as f: + caption = json.load(f) +input_image = Image.open(os.path.join(base, "ti2v_first_frame.png")).convert("RGB") + +pipe = LingBotVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="refiner/diffusion_pytorch_model*.safetensors", **vram_config), + ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern="*.safetensors", **vram_config), + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="vae/diffusion_pytorch_model.safetensors", **vram_config), + ], + processor_config=ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern=""), + vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 5, +) + +input_video = VideoData(os.path.join(base, "video_1.mp4"), height=1088, width=1920) +video = pipe( + prompt=caption, + negative_prompt=pipe.default_negative_prompt, + input_image=input_image, + input_video=input_video, + height=1088, width=1920, num_frames=81, + num_inference_steps=8, cfg_scale=3.0, + t_thresh=0.85, sigma_tail_steps=2, + seed=0, +) +save_video(video, "video_lingbot-video-moe-30b-a3b_ti2v_refined.mp4", fps=15, quality=10) diff --git a/examples/lingbot_video/model_inference_low_vram/lingbot-video-dense-1.3b_t2i.py b/examples/lingbot_video/model_inference_low_vram/lingbot-video-dense-1.3b_t2i.py new file mode 100644 index 0000000000000000000000000000000000000000..5033faccf706a5896a690d835d4749b08fa12a7c --- /dev/null +++ b/examples/lingbot_video/model_inference_low_vram/lingbot-video-dense-1.3b_t2i.py @@ -0,0 +1,44 @@ +import json +import torch +from diffsynth.pipelines.lingbot_video import LingBotVideoPipeline, ModelConfig +from modelscope import dataset_snapshot_download + +vram_config = { + "offload_dtype": "disk", + "offload_device": "disk", + "onload_dtype": torch.float8_e4m3fn, + "onload_device": "cpu", + "preparing_dtype": torch.float8_e4m3fn, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} + +pipe = LingBotVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="Robbyant/lingbot-video-dense-1.3b", origin_file_pattern="transformer/diffusion_pytorch_model.safetensors", **vram_config), + ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern="*.safetensors", **vram_config), + ModelConfig(model_id="Robbyant/lingbot-video-dense-1.3b", origin_file_pattern="vae/diffusion_pytorch_model.safetensors", **vram_config), + ], + processor_config=ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern=""), + vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 0.5, +) + +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/diffsynth_example_dataset", + local_dir="data/diffsynth_example_dataset", + allow_file_pattern="lingbot_video/lingbot-video-dense-1.3b_t2i/*", +) +with open("data/diffsynth_example_dataset/lingbot_video/lingbot-video-dense-1.3b_t2i/t2i_example.json", "r", encoding="utf-8") as f: + caption = json.load(f) + +frames = pipe( + prompt=caption, + negative_prompt=pipe.default_negative_prompt_image, + height=480, width=832, num_frames=1, + num_inference_steps=40, cfg_scale=3.0, + seed=0, +) +frames[0].save("image_lingbot-video-dense-1.3b_t2i_low_vram.png") diff --git a/examples/lingbot_video/model_inference_low_vram/lingbot-video-dense-1.3b_t2v.py b/examples/lingbot_video/model_inference_low_vram/lingbot-video-dense-1.3b_t2v.py new file mode 100644 index 0000000000000000000000000000000000000000..f66428f506c3f4cf15f05ed0fe65c05e37db82de --- /dev/null +++ b/examples/lingbot_video/model_inference_low_vram/lingbot-video-dense-1.3b_t2v.py @@ -0,0 +1,58 @@ +import torch +import json +from diffsynth.utils.data import save_video, VideoData +from diffsynth.pipelines.lingbot_video import LingBotVideoPipeline, ModelConfig +from modelscope import dataset_snapshot_download + +vram_config = { + "offload_dtype": "disk", + "offload_device": "disk", + "onload_dtype": torch.float8_e4m3fn, + "onload_device": "cpu", + "preparing_dtype": torch.float8_e4m3fn, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} + +pipe = LingBotVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="Robbyant/lingbot-video-dense-1.3b", origin_file_pattern="transformer/diffusion_pytorch_model.safetensors", **vram_config), + ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern="*.safetensors", **vram_config), + ModelConfig(model_id="Robbyant/lingbot-video-dense-1.3b", origin_file_pattern="vae/diffusion_pytorch_model.safetensors", **vram_config), + ], + processor_config=ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern=""), + vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 0.5, +) + +# --- Text-to-video ------------------------------------------------------------------- +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/diffsynth_example_dataset", + local_dir="data/diffsynth_example_dataset", + allow_file_pattern="lingbot_video/lingbot-video-dense-1.3b_t2v/*", +) +with open("data/diffsynth_example_dataset/lingbot_video/lingbot-video-dense-1.3b_t2v/t2v_example_1.json", "r", encoding="utf-8") as f: + caption = json.load(f) + +video = pipe( + prompt=caption, + negative_prompt=pipe.default_negative_prompt, + height=480, width=832, num_frames=81, + num_inference_steps=40, cfg_scale=3.0, + seed=0, +) +save_video(video, "video_lingbot-video-dense-1.3b_t2v.mp4", fps=15, quality=10) + +# --- Video-to-video ------------------------------------------------------------------ +input_video = VideoData("video_lingbot-video-dense-1.3b_t2v.mp4", height=480, width=832) +video = pipe( + prompt=caption, + negative_prompt=pipe.default_negative_prompt, + input_video=input_video, denoising_strength=0.7, + height=480, width=832, num_frames=81, + num_inference_steps=40, cfg_scale=3.0, + seed=1, +) +save_video(video, "video_lingbot-video-dense-1.3b_v2v.mp4", fps=15, quality=10) diff --git a/examples/lingbot_video/model_inference_low_vram/lingbot-video-dense-1.3b_ti2v.py b/examples/lingbot_video/model_inference_low_vram/lingbot-video-dense-1.3b_ti2v.py new file mode 100644 index 0000000000000000000000000000000000000000..cc23ece1b40e11251b39df6ca94e6c0abd33ef1f --- /dev/null +++ b/examples/lingbot_video/model_inference_low_vram/lingbot-video-dense-1.3b_ti2v.py @@ -0,0 +1,51 @@ +import os +import json +import torch +from PIL import Image +from diffsynth.utils.data import save_video +from diffsynth.pipelines.lingbot_video import LingBotVideoPipeline, ModelConfig +from modelscope import dataset_snapshot_download + +vram_config = { + "offload_dtype": "disk", + "offload_device": "disk", + "onload_dtype": torch.float8_e4m3fn, + "onload_device": "cpu", + "preparing_dtype": torch.float8_e4m3fn, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} + +pipe = LingBotVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="Robbyant/lingbot-video-dense-1.3b", origin_file_pattern="transformer/diffusion_pytorch_model.safetensors", **vram_config), + ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern="*.safetensors", **vram_config), + ModelConfig(model_id="Robbyant/lingbot-video-dense-1.3b", origin_file_pattern="vae/diffusion_pytorch_model.safetensors", **vram_config), + ], + processor_config=ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern=""), + vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 0.5, +) + +# The condition first frame and its paired caption ship in the example dataset. +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/diffsynth_example_dataset", + local_dir="data/diffsynth_example_dataset", + allow_file_pattern="lingbot_video/lingbot-video-dense-1.3b_ti2v/*", +) +base = "data/diffsynth_example_dataset/lingbot_video/lingbot-video-dense-1.3b_ti2v" +with open(os.path.join(base, "ti2v_example.json"), "r", encoding="utf-8") as f: + caption = json.load(f) +input_image = Image.open(os.path.join(base, "ti2v_first_frame.png")).convert("RGB") + +video = pipe( + prompt=caption, + negative_prompt=pipe.default_negative_prompt, + input_image=input_image, + height=480, width=832, num_frames=81, + num_inference_steps=40, cfg_scale=3.0, + seed=0, +) +save_video(video, "video_lingbot-video-dense-1.3b_ti2v_low_vram.mp4", fps=15, quality=10) diff --git a/examples/lingbot_video/model_inference_low_vram/lingbot-video-moe-30b-a3b_t2i.py b/examples/lingbot_video/model_inference_low_vram/lingbot-video-moe-30b-a3b_t2i.py new file mode 100644 index 0000000000000000000000000000000000000000..92cec75d5bde82af39d043705a8261b081d71232 --- /dev/null +++ b/examples/lingbot_video/model_inference_low_vram/lingbot-video-moe-30b-a3b_t2i.py @@ -0,0 +1,44 @@ +import json +import torch +from diffsynth.pipelines.lingbot_video import LingBotVideoPipeline, ModelConfig +from modelscope import dataset_snapshot_download + +vram_config = { + "offload_dtype": "disk", + "offload_device": "disk", + "onload_dtype": torch.float8_e4m3fn, + "onload_device": "cpu", + "preparing_dtype": torch.float8_e4m3fn, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} + +pipe = LingBotVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="transformer/diffusion_pytorch_model*.safetensors", **vram_config), + ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern="*.safetensors", **vram_config), + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="vae/diffusion_pytorch_model.safetensors", **vram_config), + ], + processor_config=ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern=""), + vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 0.5, +) + +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/diffsynth_example_dataset", + local_dir="data/diffsynth_example_dataset", + allow_file_pattern="lingbot_video/lingbot-video-moe-30b-a3b_t2i/*", +) +with open("data/diffsynth_example_dataset/lingbot_video/lingbot-video-moe-30b-a3b_t2i/t2i_example.json", "r", encoding="utf-8") as f: + caption = json.load(f) + +frames = pipe( + prompt=caption, + negative_prompt=pipe.default_negative_prompt_image, + height=480, width=832, num_frames=1, + num_inference_steps=40, cfg_scale=3.0, + seed=0, +) +frames[0].save("image_lingbot-video-moe-30b-a3b_t2i_low_vram.png") diff --git a/examples/lingbot_video/model_inference_low_vram/lingbot-video-moe-30b-a3b_t2v.py b/examples/lingbot_video/model_inference_low_vram/lingbot-video-moe-30b-a3b_t2v.py new file mode 100644 index 0000000000000000000000000000000000000000..652569c1665d7024547c96aaf1282b2f75381c9d --- /dev/null +++ b/examples/lingbot_video/model_inference_low_vram/lingbot-video-moe-30b-a3b_t2v.py @@ -0,0 +1,58 @@ +import torch +import json +from diffsynth.utils.data import save_video, VideoData +from diffsynth.pipelines.lingbot_video import LingBotVideoPipeline, ModelConfig +from modelscope import dataset_snapshot_download + +vram_config = { + "offload_dtype": "disk", + "offload_device": "disk", + "onload_dtype": torch.float8_e4m3fn, + "onload_device": "cpu", + "preparing_dtype": torch.float8_e4m3fn, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} + +pipe = LingBotVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="transformer/diffusion_pytorch_model*.safetensors", **vram_config), + ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern="*.safetensors", **vram_config), + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="vae/diffusion_pytorch_model.safetensors", **vram_config), + ], + processor_config=ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern=""), + vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 0.5, +) + +# --- Text-to-video ------------------------------------------------------------------- +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/diffsynth_example_dataset", + local_dir="data/diffsynth_example_dataset", + allow_file_pattern="lingbot_video/lingbot-video-moe-30b-a3b_t2v/*", +) +with open("data/diffsynth_example_dataset/lingbot_video/lingbot-video-moe-30b-a3b_t2v/t2v_example_1.json", "r", encoding="utf-8") as f: + caption = json.load(f) + +video = pipe( + prompt=caption, + negative_prompt=pipe.default_negative_prompt, + height=480, width=832, num_frames=81, + num_inference_steps=40, cfg_scale=3.0, + seed=0, +) +save_video(video, "video_lingbot-video-moe-30b-a3b_t2v.mp4", fps=15, quality=10) + +# --- Video-to-video ------------------------------------------------------------------ +input_video = VideoData("video_lingbot-video-moe-30b-a3b_t2v.mp4", height=480, width=832) +video = pipe( + prompt=caption, + negative_prompt=pipe.default_negative_prompt, + input_video=input_video, denoising_strength=0.7, + height=480, width=832, num_frames=81, + num_inference_steps=40, cfg_scale=3.0, + seed=1, +) +save_video(video, "video_lingbot-video-moe-30b-a3b_v2v.mp4", fps=15, quality=10) diff --git a/examples/lingbot_video/model_inference_low_vram/lingbot-video-moe-30b-a3b_t2v_refiner.py b/examples/lingbot_video/model_inference_low_vram/lingbot-video-moe-30b-a3b_t2v_refiner.py new file mode 100644 index 0000000000000000000000000000000000000000..450bedfe46200b081bf763f352f519441b69582a --- /dev/null +++ b/examples/lingbot_video/model_inference_low_vram/lingbot-video-moe-30b-a3b_t2v_refiner.py @@ -0,0 +1,49 @@ +import torch +import json +from diffsynth.utils.data import save_video, VideoData +from diffsynth.pipelines.lingbot_video import LingBotVideoPipeline, ModelConfig +from modelscope import dataset_snapshot_download + +vram_config = { + "offload_dtype": "disk", + "offload_device": "disk", + "onload_dtype": torch.float8_e4m3fn, + "onload_device": "cpu", + "preparing_dtype": torch.float8_e4m3fn, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} + +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/diffsynth_example_dataset", + local_dir="data/diffsynth_example_dataset", + allow_file_pattern="lingbot_video/lingbot-video-dense-1.3b_t2v/*", +) +base = "data/diffsynth_example_dataset/lingbot_video/lingbot-video-dense-1.3b_t2v" +with open(f"{base}/t2v_example_1.json", "r", encoding="utf-8") as f: + caption = json.load(f) + +pipe = LingBotVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="refiner/diffusion_pytorch_model*.safetensors", **vram_config), + ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern="*.safetensors", **vram_config), + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="vae/diffusion_pytorch_model.safetensors", **vram_config), + ], + processor_config=ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern=""), + vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 5, +) + +input_video = VideoData(f"{base}/video_1.mp4", height=1088, width=1920) +video = pipe( + prompt=caption, + negative_prompt=pipe.default_negative_prompt, + input_video=input_video, + height=1088, width=1920, num_frames=81, + num_inference_steps=8, cfg_scale=3.0, + t_thresh=0.85, sigma_tail_steps=2, + seed=0, +) +save_video(video, "video_lingbot-video-moe-30b-a3b_t2v_refined.mp4", fps=15, quality=10) diff --git a/examples/lingbot_video/model_inference_low_vram/lingbot-video-moe-30b-a3b_ti2v.py b/examples/lingbot_video/model_inference_low_vram/lingbot-video-moe-30b-a3b_ti2v.py new file mode 100644 index 0000000000000000000000000000000000000000..7b8de30d47dc60733e78797c21b280bb6e46f1a7 --- /dev/null +++ b/examples/lingbot_video/model_inference_low_vram/lingbot-video-moe-30b-a3b_ti2v.py @@ -0,0 +1,51 @@ +import os +import json +import torch +from PIL import Image +from diffsynth.utils.data import save_video +from diffsynth.pipelines.lingbot_video import LingBotVideoPipeline, ModelConfig +from modelscope import dataset_snapshot_download + +vram_config = { + "offload_dtype": "disk", + "offload_device": "disk", + "onload_dtype": torch.float8_e4m3fn, + "onload_device": "cpu", + "preparing_dtype": torch.float8_e4m3fn, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} + +pipe = LingBotVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="transformer/diffusion_pytorch_model*.safetensors", **vram_config), + ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern="*.safetensors", **vram_config), + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="vae/diffusion_pytorch_model.safetensors", **vram_config), + ], + processor_config=ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern=""), + vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 0.5, +) + +# The condition first frame and its paired caption ship in the example dataset. +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/diffsynth_example_dataset", + local_dir="data/diffsynth_example_dataset", + allow_file_pattern="lingbot_video/lingbot-video-moe-30b-a3b_ti2v/*", +) +base = "data/diffsynth_example_dataset/lingbot_video/lingbot-video-moe-30b-a3b_ti2v" +with open(os.path.join(base, "ti2v_example.json"), "r", encoding="utf-8") as f: + caption = json.load(f) +input_image = Image.open(os.path.join(base, "ti2v_first_frame.png")).convert("RGB") + +video = pipe( + prompt=caption, + negative_prompt=pipe.default_negative_prompt, + input_image=input_image, + height=480, width=832, num_frames=81, + num_inference_steps=40, cfg_scale=3.0, + seed=0, +) +save_video(video, "video_lingbot-video-moe-30b-a3b_ti2v_low_vram.mp4", fps=15, quality=10) diff --git a/examples/lingbot_video/model_inference_low_vram/lingbot-video-moe-30b-a3b_ti2v_refiner.py b/examples/lingbot_video/model_inference_low_vram/lingbot-video-moe-30b-a3b_ti2v_refiner.py new file mode 100644 index 0000000000000000000000000000000000000000..0323922fc4afbc2e9a0fbf26374be0a5c0c880b0 --- /dev/null +++ b/examples/lingbot_video/model_inference_low_vram/lingbot-video-moe-30b-a3b_ti2v_refiner.py @@ -0,0 +1,53 @@ +import os +import json +import torch +from PIL import Image +from diffsynth.utils.data import save_video, VideoData +from diffsynth.pipelines.lingbot_video import LingBotVideoPipeline, ModelConfig +from modelscope import dataset_snapshot_download + +vram_config = { + "offload_dtype": "disk", + "offload_device": "disk", + "onload_dtype": torch.float8_e4m3fn, + "onload_device": "cpu", + "preparing_dtype": torch.float8_e4m3fn, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} + +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/diffsynth_example_dataset", + local_dir="data/diffsynth_example_dataset", + allow_file_pattern="lingbot_video/lingbot-video-dense-1.3b_ti2v/*", +) +base = "data/diffsynth_example_dataset/lingbot_video/lingbot-video-dense-1.3b_ti2v" +with open(os.path.join(base, "ti2v_example.json"), "r", encoding="utf-8") as f: + caption = json.load(f) +input_image = Image.open(os.path.join(base, "ti2v_first_frame.png")).convert("RGB") + +pipe = LingBotVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="refiner/diffusion_pytorch_model*.safetensors", **vram_config), + ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern="*.safetensors", **vram_config), + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="vae/diffusion_pytorch_model.safetensors", **vram_config), + ], + processor_config=ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern=""), + vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 5, +) + +input_video = VideoData(os.path.join(base, "video_1.mp4"), height=1088, width=1920) +video = pipe( + prompt=caption, + negative_prompt=pipe.default_negative_prompt, + input_image=input_image, + input_video=input_video, + height=1088, width=1920, num_frames=81, + num_inference_steps=8, cfg_scale=3.0, + t_thresh=0.85, sigma_tail_steps=2, + seed=0, +) +save_video(video, "video_lingbot-video-moe-30b-a3b_ti2v_refined.mp4", fps=15, quality=10) diff --git a/examples/lingbot_video/model_training/full/accelerate_config_moe.yaml b/examples/lingbot_video/model_training/full/accelerate_config_moe.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3875a9da2354631046baf19e61a9d5ab1d8d6aca --- /dev/null +++ b/examples/lingbot_video/model_training/full/accelerate_config_moe.yaml @@ -0,0 +1,22 @@ +compute_environment: LOCAL_MACHINE +debug: false +deepspeed_config: + gradient_accumulation_steps: 1 + offload_optimizer_device: cpu + offload_param_device: cpu + zero3_init_flag: false + zero_stage: 2 +distributed_type: DEEPSPEED +downcast_bf16: 'no' +enable_cpu_affinity: false +machine_rank: 0 +main_training_function: main +mixed_precision: bf16 +num_machines: 1 +num_processes: 8 +rdzv_backend: static +same_network: true +tpu_env: [] +tpu_use_cluster: false +tpu_use_sudo: false +use_cpu: false diff --git a/examples/lingbot_video/model_training/full/lingbot-video-dense-1.3b_t2v.sh b/examples/lingbot_video/model_training/full/lingbot-video-dense-1.3b_t2v.sh new file mode 100644 index 0000000000000000000000000000000000000000..fde8f20382e1d4b0d04a3cfc7a0cf296251f8aa4 --- /dev/null +++ b/examples/lingbot_video/model_training/full/lingbot-video-dense-1.3b_t2v.sh @@ -0,0 +1,18 @@ +modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "lingbot_video/lingbot-video-dense-1.3b_t2v/*" --local_dir ./data/diffsynth_example_dataset + +accelerate launch examples/lingbot_video/model_training/train.py \ + --dataset_base_path data/diffsynth_example_dataset/lingbot_video/lingbot-video-dense-1.3b_t2v \ + --dataset_metadata_path data/diffsynth_example_dataset/lingbot_video/lingbot-video-dense-1.3b_t2v/metadata.json \ + --data_file_keys "video" \ + --height 480 \ + --width 832 \ + --num_frames 81 \ + --dataset_repeat 50 \ + --model_id_with_origin_paths "Robbyant/lingbot-video-dense-1.3b:transformer/diffusion_pytorch_model.safetensors,Qwen/Qwen3-VL-4B-Instruct:*.safetensors,Robbyant/lingbot-video-dense-1.3b:vae/diffusion_pytorch_model.safetensors" \ + --processor_path "Qwen/Qwen3-VL-4B-Instruct:" \ + --learning_rate 1e-5 \ + --num_epochs 2 \ + --remove_prefix_in_ckpt "pipe.dit." \ + --output_path "./models/train/lingbot-video-dense-1.3b_t2v_full" \ + --trainable_models "dit" \ + --use_gradient_checkpointing diff --git a/examples/lingbot_video/model_training/full/lingbot-video-dense-1.3b_ti2v.sh b/examples/lingbot_video/model_training/full/lingbot-video-dense-1.3b_ti2v.sh new file mode 100644 index 0000000000000000000000000000000000000000..56d1394df311b86e95b66c4085d3921a42aee31a --- /dev/null +++ b/examples/lingbot_video/model_training/full/lingbot-video-dense-1.3b_ti2v.sh @@ -0,0 +1,19 @@ +modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "lingbot_video/lingbot-video-dense-1.3b_ti2v/*" --local_dir ./data/diffsynth_example_dataset + +accelerate launch examples/lingbot_video/model_training/train.py \ + --dataset_base_path data/diffsynth_example_dataset/lingbot_video/lingbot-video-dense-1.3b_ti2v \ + --dataset_metadata_path data/diffsynth_example_dataset/lingbot_video/lingbot-video-dense-1.3b_ti2v/metadata.json \ + --data_file_keys "video" \ + --height 480 \ + --width 832 \ + --num_frames 81 \ + --first_frame_as_condition \ + --dataset_repeat 50 \ + --model_id_with_origin_paths "Robbyant/lingbot-video-dense-1.3b:transformer/diffusion_pytorch_model.safetensors,Qwen/Qwen3-VL-4B-Instruct:*.safetensors,Robbyant/lingbot-video-dense-1.3b:vae/diffusion_pytorch_model.safetensors" \ + --processor_path "Qwen/Qwen3-VL-4B-Instruct:" \ + --learning_rate 1e-5 \ + --num_epochs 2 \ + --remove_prefix_in_ckpt "pipe.dit." \ + --output_path "./models/train/lingbot-video-dense-1.3b_ti2v_full" \ + --trainable_models "dit" \ + --use_gradient_checkpointing diff --git a/examples/lingbot_video/model_training/full/lingbot-video-moe-30b-a3b_t2v.sh b/examples/lingbot_video/model_training/full/lingbot-video-moe-30b-a3b_t2v.sh new file mode 100644 index 0000000000000000000000000000000000000000..0c242886ee4498aa320da5206e3f7468d9160a38 --- /dev/null +++ b/examples/lingbot_video/model_training/full/lingbot-video-moe-30b-a3b_t2v.sh @@ -0,0 +1,19 @@ +modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "lingbot_video/lingbot-video-moe-30b-a3b_t2v/*" --local_dir ./data/diffsynth_example_dataset + +accelerate launch --config_file examples/lingbot_video/model_training/full/accelerate_config_moe.yaml examples/lingbot_video/model_training/train.py \ + --dataset_base_path data/diffsynth_example_dataset/lingbot_video/lingbot-video-moe-30b-a3b_t2v \ + --dataset_metadata_path data/diffsynth_example_dataset/lingbot_video/lingbot-video-moe-30b-a3b_t2v/metadata.json \ + --data_file_keys "video" \ + --height 480 \ + --width 832 \ + --num_frames 81 \ + --dataset_repeat 50 \ + --model_id_with_origin_paths "Robbyant/lingbot-video-moe-30b-a3b:transformer/diffusion_pytorch_model*.safetensors,Qwen/Qwen3-VL-4B-Instruct:*.safetensors,Robbyant/lingbot-video-moe-30b-a3b:vae/diffusion_pytorch_model.safetensors" \ + --processor_path "Qwen/Qwen3-VL-4B-Instruct:" \ + --learning_rate 1e-5 \ + --num_epochs 2 \ + --remove_prefix_in_ckpt "pipe.dit." \ + --output_path "./models/train/lingbot-video-moe-30b-a3b_t2v_full" \ + --trainable_models "dit" \ + --use_gradient_checkpointing \ + --use_gradient_checkpointing_offload diff --git a/examples/lingbot_video/model_training/full/lingbot-video-moe-30b-a3b_ti2v.sh b/examples/lingbot_video/model_training/full/lingbot-video-moe-30b-a3b_ti2v.sh new file mode 100644 index 0000000000000000000000000000000000000000..aebed338922d3cb730af434bd01dde4848ae3c63 --- /dev/null +++ b/examples/lingbot_video/model_training/full/lingbot-video-moe-30b-a3b_ti2v.sh @@ -0,0 +1,20 @@ +modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "lingbot_video/lingbot-video-moe-30b-a3b_ti2v/*" --local_dir ./data/diffsynth_example_dataset + +accelerate launch --config_file examples/lingbot_video/model_training/full/accelerate_config_moe.yaml examples/lingbot_video/model_training/train.py \ + --dataset_base_path data/diffsynth_example_dataset/lingbot_video/lingbot-video-moe-30b-a3b_ti2v \ + --dataset_metadata_path data/diffsynth_example_dataset/lingbot_video/lingbot-video-moe-30b-a3b_ti2v/metadata.json \ + --data_file_keys "video" \ + --height 480 \ + --width 832 \ + --num_frames 81 \ + --first_frame_as_condition \ + --dataset_repeat 50 \ + --model_id_with_origin_paths "Robbyant/lingbot-video-moe-30b-a3b:transformer/diffusion_pytorch_model*.safetensors,Qwen/Qwen3-VL-4B-Instruct:*.safetensors,Robbyant/lingbot-video-moe-30b-a3b:vae/diffusion_pytorch_model.safetensors" \ + --processor_path "Qwen/Qwen3-VL-4B-Instruct:" \ + --learning_rate 1e-5 \ + --num_epochs 2 \ + --remove_prefix_in_ckpt "pipe.dit." \ + --output_path "./models/train/lingbot-video-moe-30b-a3b_ti2v_full" \ + --trainable_models "dit" \ + --use_gradient_checkpointing \ + --use_gradient_checkpointing_offload diff --git a/examples/lingbot_video/model_training/lora/lingbot-video-dense-1.3b_t2v.sh b/examples/lingbot_video/model_training/lora/lingbot-video-dense-1.3b_t2v.sh new file mode 100644 index 0000000000000000000000000000000000000000..91728fae028227354d85a5b7a3f9de95565756c3 --- /dev/null +++ b/examples/lingbot_video/model_training/lora/lingbot-video-dense-1.3b_t2v.sh @@ -0,0 +1,20 @@ +modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "lingbot_video/lingbot-video-dense-1.3b_t2v/*" --local_dir ./data/diffsynth_example_dataset + +accelerate launch examples/lingbot_video/model_training/train.py \ + --dataset_base_path data/diffsynth_example_dataset/lingbot_video/lingbot-video-dense-1.3b_t2v \ + --dataset_metadata_path data/diffsynth_example_dataset/lingbot_video/lingbot-video-dense-1.3b_t2v/metadata.json \ + --data_file_keys "video" \ + --height 480 \ + --width 832 \ + --num_frames 81 \ + --dataset_repeat 50 \ + --model_id_with_origin_paths "Robbyant/lingbot-video-dense-1.3b:transformer/diffusion_pytorch_model.safetensors,Qwen/Qwen3-VL-4B-Instruct:*.safetensors,Robbyant/lingbot-video-dense-1.3b:vae/diffusion_pytorch_model.safetensors" \ + --processor_path "Qwen/Qwen3-VL-4B-Instruct:" \ + --learning_rate 1e-4 \ + --num_epochs 5 \ + --remove_prefix_in_ckpt "pipe.dit." \ + --output_path "./models/train/lingbot-video-dense-1.3b_t2v_lora" \ + --lora_base_model "dit" \ + --lora_target_modules "to_q,to_k,to_v,to_out" \ + --lora_rank 32 \ + --use_gradient_checkpointing diff --git a/examples/lingbot_video/model_training/lora/lingbot-video-dense-1.3b_ti2v.sh b/examples/lingbot_video/model_training/lora/lingbot-video-dense-1.3b_ti2v.sh new file mode 100644 index 0000000000000000000000000000000000000000..ec1983476d8fd88a282f2aad5e07f104da9f8940 --- /dev/null +++ b/examples/lingbot_video/model_training/lora/lingbot-video-dense-1.3b_ti2v.sh @@ -0,0 +1,21 @@ +modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "lingbot_video/lingbot-video-dense-1.3b_ti2v/*" --local_dir ./data/diffsynth_example_dataset + +accelerate launch examples/lingbot_video/model_training/train.py \ + --dataset_base_path data/diffsynth_example_dataset/lingbot_video/lingbot-video-dense-1.3b_ti2v \ + --dataset_metadata_path data/diffsynth_example_dataset/lingbot_video/lingbot-video-dense-1.3b_ti2v/metadata.json \ + --data_file_keys "video" \ + --height 480 \ + --width 832 \ + --num_frames 81 \ + --first_frame_as_condition \ + --dataset_repeat 50 \ + --model_id_with_origin_paths "Robbyant/lingbot-video-dense-1.3b:transformer/diffusion_pytorch_model.safetensors,Qwen/Qwen3-VL-4B-Instruct:*.safetensors,Robbyant/lingbot-video-dense-1.3b:vae/diffusion_pytorch_model.safetensors" \ + --processor_path "Qwen/Qwen3-VL-4B-Instruct:" \ + --learning_rate 1e-4 \ + --num_epochs 5 \ + --remove_prefix_in_ckpt "pipe.dit." \ + --output_path "./models/train/lingbot-video-dense-1.3b_ti2v_lora" \ + --lora_base_model "dit" \ + --lora_target_modules "to_q,to_k,to_v,to_out" \ + --lora_rank 32 \ + --use_gradient_checkpointing diff --git a/examples/lingbot_video/model_training/lora/lingbot-video-moe-30b-a3b_t2v.sh b/examples/lingbot_video/model_training/lora/lingbot-video-moe-30b-a3b_t2v.sh new file mode 100644 index 0000000000000000000000000000000000000000..4ce53a786a7ea5e8c75327da2e0ac5aaff977e5b --- /dev/null +++ b/examples/lingbot_video/model_training/lora/lingbot-video-moe-30b-a3b_t2v.sh @@ -0,0 +1,20 @@ +modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "lingbot_video/lingbot-video-moe-30b-a3b_t2v/*" --local_dir ./data/diffsynth_example_dataset + +accelerate launch --config_file examples/lingbot_video/model_training/full/accelerate_config_moe.yaml examples/lingbot_video/model_training/train.py \ + --dataset_base_path data/diffsynth_example_dataset/lingbot_video/lingbot-video-moe-30b-a3b_t2v \ + --dataset_metadata_path data/diffsynth_example_dataset/lingbot_video/lingbot-video-moe-30b-a3b_t2v/metadata.json \ + --data_file_keys "video" \ + --height 480 \ + --width 832 \ + --num_frames 81 \ + --dataset_repeat 50 \ + --model_id_with_origin_paths "Robbyant/lingbot-video-moe-30b-a3b:transformer/diffusion_pytorch_model*.safetensors,Qwen/Qwen3-VL-4B-Instruct:*.safetensors,Robbyant/lingbot-video-moe-30b-a3b:vae/diffusion_pytorch_model.safetensors" \ + --processor_path "Qwen/Qwen3-VL-4B-Instruct:" \ + --learning_rate 1e-4 \ + --num_epochs 5 \ + --remove_prefix_in_ckpt "pipe.dit." \ + --output_path "./models/train/lingbot-video-moe-30b-a3b_t2v_lora" \ + --lora_base_model "dit" \ + --lora_target_modules "to_q,to_k,to_v,to_out" \ + --lora_rank 32 \ + --use_gradient_checkpointing diff --git a/examples/lingbot_video/model_training/lora/lingbot-video-moe-30b-a3b_ti2v.sh b/examples/lingbot_video/model_training/lora/lingbot-video-moe-30b-a3b_ti2v.sh new file mode 100644 index 0000000000000000000000000000000000000000..7fd74180e28ea00d40161c2b14facf1b46a3de6c --- /dev/null +++ b/examples/lingbot_video/model_training/lora/lingbot-video-moe-30b-a3b_ti2v.sh @@ -0,0 +1,21 @@ +modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "lingbot_video/lingbot-video-moe-30b-a3b_ti2v/*" --local_dir ./data/diffsynth_example_dataset + +accelerate launch --config_file examples/lingbot_video/model_training/full/accelerate_config_moe.yaml examples/lingbot_video/model_training/train.py \ + --dataset_base_path data/diffsynth_example_dataset/lingbot_video/lingbot-video-moe-30b-a3b_ti2v \ + --dataset_metadata_path data/diffsynth_example_dataset/lingbot_video/lingbot-video-moe-30b-a3b_ti2v/metadata.json \ + --data_file_keys "video" \ + --height 480 \ + --width 832 \ + --num_frames 81 \ + --first_frame_as_condition \ + --dataset_repeat 50 \ + --model_id_with_origin_paths "Robbyant/lingbot-video-moe-30b-a3b:transformer/diffusion_pytorch_model*.safetensors,Qwen/Qwen3-VL-4B-Instruct:*.safetensors,Robbyant/lingbot-video-moe-30b-a3b:vae/diffusion_pytorch_model.safetensors" \ + --processor_path "Qwen/Qwen3-VL-4B-Instruct:" \ + --learning_rate 1e-4 \ + --num_epochs 5 \ + --remove_prefix_in_ckpt "pipe.dit." \ + --output_path "./models/train/lingbot-video-moe-30b-a3b_ti2v_lora" \ + --lora_base_model "dit" \ + --lora_target_modules "to_q,to_k,to_v,to_out" \ + --lora_rank 32 \ + --use_gradient_checkpointing diff --git a/examples/lingbot_video/model_training/scripts/prompt_rewriter.py b/examples/lingbot_video/model_training/scripts/prompt_rewriter.py new file mode 100644 index 0000000000000000000000000000000000000000..b27ff9f9eafa58252f14c121d30103033242b4e7 --- /dev/null +++ b/examples/lingbot_video/model_training/scripts/prompt_rewriter.py @@ -0,0 +1,213 @@ +"""Two-stage prompt rewriter for LingBot-Video (example-side helper). + +rewrite_prompt turns a brief idea into the structured JSON caption the DiT expects: +stage 1 expands the idea into a natural-language caption (base model), stage 2 maps it +into structured JSON (base model + stage-2 LoRA). This is a separate VLM + LoRA adapter, +not the DiT, and is not downloaded with the pipeline -- hence it lives with the examples. +TransformersBackend loads the rewriter VLM locally; make_backend also accepts any object +exposing generate(text, image, use_lora). + + from examples.lingbot_video.model_training.scripts.prompt_rewriter import rewrite_prompt + caption = rewrite_prompt("a puppy running across a meadow", mode="t2v", duration=5) + video = pipe(prompt=caption, ...) +""" + +import io +import json +import os +import re + +# Optional deps: only needed for the local rewriter backend / image loading. +try: + import requests +except ImportError: + requests = None + +try: + from PIL import Image +except ImportError: + Image = None + +try: + from json_repair import repair_json +except ImportError: + repair_json = None + +try: + from .system_prompts import ( + VIDEO_STEP1_EXPAND, VIDEO_STEP2_MAP, IMAGE_STEP1_EXPAND, IMAGE_STEP2_MAP, + ) +except ImportError: + from system_prompts import ( + VIDEO_STEP1_EXPAND, VIDEO_STEP2_MAP, IMAGE_STEP1_EXPAND, IMAGE_STEP2_MAP, + ) + + +# mode -> (step1 system prompt, step2 system prompt, feed image?, add duration?) +MODES = { + "t2v": dict(s1=VIDEO_STEP1_EXPAND, s2=VIDEO_STEP2_MAP, image=False, duration=True), + "ti2v": dict(s1=VIDEO_STEP1_EXPAND, s2=VIDEO_STEP2_MAP, image=True, duration=True), + "t2i": dict(s1=IMAGE_STEP1_EXPAND, s2=IMAGE_STEP2_MAP, image=False, duration=False), +} + + +def _has_cjk(s: str) -> bool: + return any("一" <= c <= "鿿" for c in s) + + +def load_image(src): + """Load a first-frame image from a local path / http(s) URL / PIL.Image -> RGB PIL.""" + if Image is None: + raise ImportError("loading a first-frame image requires the Pillow package.") + if isinstance(src, Image.Image): + return src.convert("RGB") + if isinstance(src, str) and re.match(r"^https?://", src): + if requests is None: + raise ImportError("fetching an image URL requires the requests package.") + return Image.open(io.BytesIO(requests.get(src, timeout=30).content)).convert("RGB") + return Image.open(src).convert("RGB") + + +def _step1_text(mode, prompt, dur): + sys = MODES[mode]["s1"] + if mode == "t2i": + return sys + "\n\nUser image prompt:\n" + prompt + dur_line = f"\n\n视频时长：{dur} 秒" if _has_cjk(prompt) else f"\n\nVideo Duration: {dur} seconds" + return sys + "\n\n" + prompt + dur_line + + +def _step2_text(mode, detailed, dur): + sys = MODES[mode]["s2"] + if mode == "t2i": + return sys + "\n\nDETAILED CAPTION:\n" + detailed + return (sys + f"\n\nVideo Duration: {dur} seconds\n\nDETAILED CAPTION:\n" + + detailed + "\n\nOutput the JSON now.") + + +def parse_json(raw): + """Parse the stage-2 output into a dict, or None. Strips any code fence, tries the + stdlib parser, then falls back to json_repair (if installed) for messy output.""" + s = (raw or "").strip() + m = re.search(r"```(?:json)?\s*(\{.*\})\s*```", s, re.DOTALL) + if m: + s = m.group(1) + try: + obj = json.loads(s) + if isinstance(obj, dict): + return obj + except Exception: + pass + if repair_json is not None: + try: + obj = repair_json(s, return_objects=True) + return obj if isinstance(obj, dict) else None + except Exception: + return None + return None + + +def save_caption(result, duration, path): + """Save as {"caption": , "duration": } -- the prompt.json the + pipeline consumes. duration is integer seconds for T2V/TI2V, None for T2I.""" + dur = int(round(duration)) if MODES[result["mode"]]["duration"] else None + with open(path, "w", encoding="utf-8") as f: + json.dump({"caption": result["json"], "duration": dur}, f, ensure_ascii=False, indent=2) + return path + + +class TransformersBackend: + """Local rewriter VLM + LoRA adapter (peft): stage1 = base (adapter disabled), + stage2 = base + LoRA. Set base/adapter (or REWRITER_BASE_MODEL/REWRITER_ADAPTER).""" + + def __init__(self, base=None, adapter=None, device="auto", max_new_tokens=6144): + import contextlib + import torch + from peft import PeftModel + from transformers import AutoModelForImageTextToText, AutoProcessor + + self._contextlib = contextlib + self._torch = torch + + base = base or os.environ.get("REWRITER_BASE_MODEL", "") + adapter = adapter or os.environ.get("REWRITER_ADAPTER", "") + if not base or not adapter: + raise ValueError( + "Set the rewriter base and adapter paths via base=/adapter= or the " + "REWRITER_BASE_MODEL / REWRITER_ADAPTER environment variables." + ) + self.processor = AutoProcessor.from_pretrained(base, trust_remote_code=True) + model = AutoModelForImageTextToText.from_pretrained( + base, torch_dtype=torch.bfloat16, device_map=device, trust_remote_code=True) + self.model = PeftModel.from_pretrained(model, adapter).eval() + self.max_new_tokens = max_new_tokens + + def generate(self, text, image, use_lora): + torch = self._torch + content = ([{"type": "image", "image": image}] if image is not None else []) \ + + [{"type": "text", "text": text}] + messages = [{"role": "user", "content": content}] + chat = self.processor.apply_chat_template( + messages, tokenize=False, add_generation_prompt=True, enable_thinking=False) + inputs = self.processor( + text=[chat], images=([image] if image is not None else None), return_tensors="pt" + ).to(self.model.device) + # stage1 (expand): disable LoRA; stage2 (map): keep LoRA active. + adapter_ctx = self._contextlib.nullcontext() if use_lora else self.model.disable_adapter() + with torch.no_grad(), adapter_ctx: + out = self.model.generate(**inputs, max_new_tokens=self.max_new_tokens, do_sample=False) + gen = out[:, inputs["input_ids"].shape[1]:] + return self.processor.batch_decode(gen, skip_special_tokens=True)[0] + + +def make_backend(backend="transformers", base=None, adapter=None): + """Build a rewriter backend: "transformers" for the local TransformersBackend, or any + object already exposing generate(text, image, use_lora) -> str (e.g. a hosted endpoint).""" + if backend == "transformers": + return TransformersBackend(base, adapter) + if hasattr(backend, "generate"): + return backend + raise ValueError( + f"unknown backend: {backend!r}; pass 'transformers' or an object exposing " + "generate(text, image, use_lora)." + ) + + +class Rewriter: + """Two-stage orchestrator. The backend implements ``generate(text, image, use_lora) -> str``.""" + + def __init__(self, backend): + self.backend = backend + + def rewrite(self, prompt, mode="t2v", first_frame=None, duration=5): + if mode not in MODES: + raise ValueError(f"mode must be one of {list(MODES)}") + cfg = MODES[mode] + dur = int(round(duration)) + img = None + if cfg["image"]: + if first_frame is None: + raise ValueError(f"{mode} requires first_frame (path / URL / PIL.Image)") + img = load_image(first_frame) + # stage 1: EXPAND -- base model (no LoRA) + detailed = self.backend.generate(_step1_text(mode, prompt, dur), img, use_lora=False).strip() + # stage 2: MAP -- base + LoRA + raw = self.backend.generate(_step2_text(mode, detailed, dur), img, use_lora=True).strip() + return {"mode": mode, "detailed": detailed, "json": parse_json(raw), "json_raw": raw} + + +def rewrite_prompt(prompt, mode="t2v", first_frame=None, duration=5, + backend="transformers", base=None, adapter=None, return_result=False): + """Rewrite a brief idea into the structured caption string the pipeline expects. + + Returns the compact-JSON caption string (ready to pass as ``pipe(prompt=...)``). + Pass ``return_result=True`` to also get the full stage-1/stage-2 dict. + """ + rw = Rewriter(make_backend(backend, base, adapter)) + result = rw.rewrite(prompt, mode=mode, first_frame=first_frame, duration=duration) + if result["json"] is not None: + caption = json.dumps(result["json"], ensure_ascii=False, separators=(",", ":")) + else: + caption = result["json_raw"] + if return_result: + return caption, result + return caption diff --git a/examples/lingbot_video/model_training/scripts/rewrite_captions.py b/examples/lingbot_video/model_training/scripts/rewrite_captions.py new file mode 100644 index 0000000000000000000000000000000000000000..84a1de7376fb0b14212560db3c88bb166726a0c7 --- /dev/null +++ b/examples/lingbot_video/model_training/scripts/rewrite_captions.py @@ -0,0 +1,91 @@ +"""Offline: rewrite a training metadata file's raw prompts into structured JSON captions. + +Run this once before training if your dataset's prompt column holds free-form prose, then +train on the rewritten metadata. The rewriter is a separate VLM + stage-2 LoRA adapter +(not the DiT); provide it via --base/--adapter or REWRITER_BASE_MODEL/REWRITER_ADAPTER. + +Usage: + python rewrite_captions.py --metadata metadata.csv --output metadata_rewritten.csv \ + --base /path/to/rewriter-base --adapter /path/to/rewriter-step2-lora --duration 5 + +Supports .csv / .json / .jsonl. Rows whose stage-2 output fails to parse keep their +original prompt and are logged, so training never silently uses a broken row. +""" + +import argparse +import json +import os + +try: + from .prompt_rewriter import Rewriter, make_backend +except ImportError: + from prompt_rewriter import Rewriter, make_backend + + +def _load_rows(path): + if path.endswith(".json"): + with open(path, "r", encoding="utf-8") as f: + return json.load(f), "json" + if path.endswith(".jsonl"): + rows = [] + with open(path, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if line: + rows.append(json.loads(line)) + return rows, "jsonl" + import pandas + df = pandas.read_csv(path) + return [df.iloc[i].to_dict() for i in range(len(df))], "csv" + + +def _save_rows(rows, path, fmt): + if fmt == "json": + with open(path, "w", encoding="utf-8") as f: + json.dump(rows, f, ensure_ascii=False, indent=2) + elif fmt == "jsonl": + with open(path, "w", encoding="utf-8") as f: + for row in rows: + f.write(json.dumps(row, ensure_ascii=False) + "\n") + else: + import pandas + pandas.DataFrame(rows).to_csv(path, index=False) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--metadata", required=True, help="input metadata (.csv/.json/.jsonl)") + ap.add_argument("--output", required=True, help="output metadata with rewritten captions") + ap.add_argument("--prompt-column", default="prompt") + ap.add_argument("--mode", default="t2v", choices=["t2v", "ti2v", "t2i"]) + ap.add_argument("--duration", type=float, default=5) + ap.add_argument("--backend", default="transformers") + ap.add_argument("--base", default=None, help="rewriter base model (or REWRITER_BASE_MODEL)") + ap.add_argument("--adapter", default=None, help="rewriter stage-2 LoRA (or REWRITER_ADAPTER)") + ap.add_argument("--first-frame-column", default=None, + help="for ti2v: column holding the first-frame image path/URL") + args = ap.parse_args() + + rows, fmt = _load_rows(args.metadata) + rewriter = Rewriter(make_backend(args.backend, args.base, args.adapter)) + + ok, failed = 0, 0 + for i, row in enumerate(rows): + raw = row.get(args.prompt_column, "") + first_frame = row.get(args.first_frame_column) if args.first_frame_column else None + result = rewriter.rewrite(raw, mode=args.mode, first_frame=first_frame, duration=args.duration) + if result["json"] is not None: + row[args.prompt_column] = json.dumps(result["json"], ensure_ascii=False, separators=(",", ":")) + ok += 1 + else: + failed += 1 + print(f"[warn] row {i}: stage-2 JSON did not parse; keeping original prompt.") + if (i + 1) % 20 == 0: + print(f" ... {i + 1}/{len(rows)} rewritten") + + _save_rows(rows, args.output, fmt) + print(f"[done] {ok} rewritten, {failed} kept-as-is -> {os.path.abspath(args.output)}") + + +if __name__ == "__main__": + main() diff --git a/examples/lingbot_video/model_training/scripts/system_prompts.py b/examples/lingbot_video/model_training/scripts/system_prompts.py new file mode 100644 index 0000000000000000000000000000000000000000..35064b564a5aca950fd1ccb1229ef073d3e7ec7d --- /dev/null +++ b/examples/lingbot_video/model_training/scripts/system_prompts.py @@ -0,0 +1,301 @@ +# LingBot-Video prompt-rewriter system prompts. +# +# Copied VERBATIM from the original lingbot-video package +# (rewriter/system_prompts.py) so the two-stage rewriter produces captions in the +# exact format the DiT was trained on. Do not paraphrase these strings — the model +# is sensitive to them. Four constants: VIDEO_STEP1_EXPAND / VIDEO_STEP2_MAP +# (text->natural caption, then caption->structured JSON) and the IMAGE_* pair. + +VIDEO_STEP1_EXPAND = ( + 'You write ONE short, natural, standalone video caption in English from a brief user video\n' + 'prompt — as if briefly recounting what happens in a real clip of that idea.\n' + '\n' + "INPUT: a short user prompt (and, when provided, the video's first frame as a visual anchor).\n" + '\n' + 'HARD LENGTH LIMIT: the output MUST be UNDER 1000 characters. Aim for roughly 400-800\n' + 'characters. Concise is a top priority.\n' + '\n' + 'STYLE — a flowing story, NOT a checklist:\n' + '- Connected, natural English prose. NO headings, NO bullets, NO field labels.\n' + '- The MAIN THREAD is what HAPPENS — lead with the action. Do NOT march subject-by-subject\n' + ' listing attributes; weave the few details you keep into the action naturally.\n' + '- One cohesive paragraph; never pad to add length.\n' + '\n' + 'WHAT TO COVER (only these):\n' + '- The scene in a brief phrase.\n' + '- Each main subject by name/what-it-is plus ONE most defining visual trait — not a full\n' + ' appearance list.\n' + '- What happens over the clip, in correct chronological order as the backbone, with an EXPLICIT\n' + ' timestamp in seconds on each action, distributed within the given video duration — e.g.\n' + ' "at 0.0s", "from 1.2s to 2.7s", "around the 3.4s mark", "finally from 4.0s to 5.0s". Span the\n' + ' actions from 0s up to (but never beyond) the stated duration; the last one ends at or before it.\n' + ' Attach a timestamp ONLY to a real action or change; mention a static/unchanging element once\n' + ' with NO timestamp — never put a whole-clip span on something that merely exists.\n' + '- A one-word shot type only if it matters; otherwise skip the camera.\n' + "- Named entities only if clearly implied — don't invent identities.\n" + '\n' + 'FAITHFUL EXPANSION:\n' + '- Stay consistent with the prompt; elaborate plausibly but never contradict it. If a first\n' + ' frame is given, ground the scene and subject in it.\n' + '- Keep actions in their natural, real-world direction — never reverse them. Commit to one\n' + " coherent interpretation; don't hedge with alternatives.\n" + '- Keep subject identities and counts consistent.\n' + '\n' + 'DOMAIN NOTES (apply whichever fits):\n' + '- Multiple events / sequence: lay out consecutive actions in STRICT chronological order as\n' + ' distinct, separated steps with approximate timing — never blur them into one vague action.\n' + '- Robot manipulation (VLA): subjects are robotic arm(s), gripper(s) and workspace objects,\n' + ' NOT people — no clothing/skin/gender/expression. When two arms are present, ALWAYS keep the\n' + ' LEFT and RIGHT arm distinct and state which arm/gripper does each motion; never swap, merge,\n' + ' or leave it ambiguous. Manipulation is STRICTLY NOT reversible.\n' + "- First-person (EGO): a head-mounted first-person view; camera motion IS the wearer's head\n" + ' movement. The agent is the camera-wearer, a PERSON shown through their own hands/arms and\n' + ' viewpoint; never a robot or external third-person subject. Stay strictly first-person.\n' + '\n' + 'Output ONLY the caption text — no preamble, no headings, no explanation.\n' + '\n' + "## EXAMPLES — one representative example per domain. When you write your own caption, match these examples' style, format, length, and timestamp convention (each action gets a numeric timestamp in seconds within the video's duration).\n" + '\n' + '### Example 1 — general\n' + 'USER PROMPT:\n' + '低角度广角镜头，镜头静止。中央大型条纹热气球缓慢上升并飘移（气球呈红橙黄绿蓝条纹，底部深色），背景中多个热气球在天空漂移。地面草地上散布着正在充气的热气球，一辆红色皮卡停放在旁。随后一辆蓝色拖车进入画面并横穿前景（覆盖蓝色防水布）。饱和色彩，硬光，日光，电影质感。\n' + '\n' + 'DETAILED CAPTION:\n' + 'At a vibrant hot air balloon festival on a sunny day, a wide shot shows a grassy field under a blue sky. A large, multi-colored striped balloon dominates the center, and from the start at 0.0 seconds until 5.3 seconds, it slowly rises and drifts slightly upward and to the right. Simultaneously during the 0.0s to 5.3s period, several small background balloons drift slowly across the sky in various directions above a cluster of grounded balloons. A red pickup truck and white van sit parked near the left side, while finally, around the 4.3s mark until 5.3s, a blue trailer enters the frame from the right and moves left across the foreground.\n' + '\n' + '### Example 2 — multi-event\n' + 'USER PROMPT:\n' + 'Subtitle: Outdoor High-Intensity Workout — All live-action, hyper-realistic, fixed shot with slight handheld unsteadiness.\n' + '\n' + 'Style: Hard daylight, high-angle wide shot, center composition, sharp shadows, athletic tension.\n' + '\n' + '1. Muscular shirtless man, black headwrap, black boxing gloves, blue digital camo pants, black combat boots. Standing on concrete surface, background includes beige wall, chain-link fence, and trash bins. Stands in a boxing stance with hands up.\n' + '\n' + '2. Drops into a deep squat, then jumps upward explosively.\n' + '\n' + '3. Lands and drops into a deep squat, repeating this explosive jump and squat landing cycle.\n' + '\n' + '4. After the final landing and squat, begins to stand up.\n' + '\n' + 'Overall movements coherent and natural, high contrast lighting, focused and athletic atmosphere.\n' + '\n' + 'DETAILED CAPTION:\n' + 'In an outdoor urban setting against a beige wall, a shirtless man wearing boxing gloves performs a workout on concrete. From 0.0s to 0.6s he stands in a boxing stance with hands up. Then from 0.6s to 1.2s he drops into a deep squat, followed by the period from 1.2s to 1.8s where he jumps upward explosively. From 1.8s to 2.4s he lands and drops into a deep squat, then from 2.4s to 3.0s jumps upward explosively again. Around the 3.0s to 3.6s mark he lands and drops into a deep squat, proceeding to jump upward explosively from 3.6s to 4.2s. From 4.2s to 4.8s he lands and drops into a deep squat once more, and finally from 4.8s to 5.0s he begins to stand up.\n' + '\n' + '### Example 3 — VLA\n' + 'USER PROMPT:\n' + '环境：自动化超市补货工作站，俯视视角，明亮均匀人工光，清晰功能化氛围。物体：左侧打开的棕色纸箱内含整齐堆叠的红色香肠包装，右侧白色矩形料箱内含红黄绿混合食品包装及带黑色把手的透明塑料隔板。机器人：双臂系统。左臂黑色机身银色底座黑色夹爪（全程静止悬停），右臂白色机身黑色夹爪腕部蓝色指示灯（活动主体）。动作：右臂向下向左移动伸入白色料箱，抓取隔板黑色把手，向右滑动隔板，释放把手，向上向右收回复位。左臂悬停于纸箱上方保持静止。相机：固定高角度俯视视角，全程静止镜头，宽画幅，超广角镜头，柔和人工光，极致细节。\n' + '\n' + 'DETAILED CAPTION:\n' + 'A top-down wide shot shows an automated workspace with a cardboard box of sausage packages on the left and a white bin on the right. A stationary black and silver left robotic arm hovers over the box, while a white and black right robotic arm operates above the bin containing a black handle. From 0.0s to 2.0s, the right arm moves downwards and to the left, reaching into the white bin as the handle remains still. Then from 2.0s to 6.0s, the right arm grasps the black handle and moves to the right, pulling the handle and sliding the divider across the bin. Finally from 6.0s to 9.2s, the right arm releases the handle and moves upwards and to the right, returning to a resting position while the handle stays stationary. The box and bin remain fixed throughout.\n' + '\n' + '### Example 4 — EGO\n' + 'USER PROMPT:\n' + 'First-person POV, brightly lit grocery store produce section, soft artificial lighting. Two rectangular bins side-by-side in front; left filled with red/yellow apples, right piled with bright orange oranges. Background shows aisles and shoppers, one in red shirt with basket. Right hand enters from bottom, reaches into right orange pile, grasps one orange, lifts it upwards and slightly left. Camera moves forward and slightly down approaching bins, then remains stable with minor panning/tilting following hand movement.\n' + '\n' + 'DETAILED CAPTION:\n' + "From a first-person perspective in a grocery store produce section, the camera approaches display bins containing apples on the left and bright oranges on the right. In the background aisles, a shopper wearing a red shirt walks from the left side of the frame towards the right between 0.0s and 2.0s, while another shopper in a striped shirt walks away from the camera down the aisle from 0.0s to 5.0s. The operator's right hand enters the frame from the bottom from 0.0s to 2.5s, then reaches into the pile of oranges and grasps one from 2.5s to 3.5s. Finally, the hand lifts the grasped orange upwards and to the left from 3.5s to 5.0s, selecting it from the cluster." +) + +VIDEO_STEP2_MAP = ( + 'You are a structuring engine. You convert a DETAILED natural-language video caption (which concisely states the scene and the full motion/timing plan) into a STRUCTURED JSON caption. Your job is FAITHFUL structural extraction ONLY: do NOT drop, alter, reorder, or re-time anything the prose states (keep every subject, every action, and every timestamp exactly), and do NOT re-plan motion; BUT the prose is brief and intentionally omits fine visual attributes and camera settings — you MUST fill those omitted fields (texture, skin tone, precise colors, relative size, pose/orientation, clothing, and all camera_info) with plausible values that stay consistent with and never contradict the prose — just map what the prose already states into the schema.\n' + '\n' + '## OUTPUT FORMAT (JSON)\n' + '\n' + '```json\n' + '{\n' + ' "comprehensive_description": {\n' + ' "scene_content_description": "(String) A detailed description focusing on scene content, subject appearance, lighting, atmosphere, narrative, and interactions between elements. Text physically printed on visual objects should be mentioned alongside the object and detailed further in prominent_elements. For standalone OCR/text elements, briefly describe their role, relative scale, font style, color, and orientation; categorize them as \'static overlays\' (like watermarks) or \'integrated scene text\' (like subtitles/scrolling text); and describe their temporal behavior if they appear or disappear. Provide an exact transcription for prominent text (preserving spelling, punctuation, and capitalization), but summarize long or dense text blocks instead of transcribing them fully. **Do not describe camera movement here**. Maximum 800 words.",\n' + ' "camera_movement_description": "(String) A detailed description focusing on camera behavior. Include camera movement types (Pan, Tilt, Zoom, Dolly, Truck, Roll), shooting angles (high/low/eye-level), shot size changes, and stability. Maximum 100 words. If the camera is essentially stationary, set to \'\'."\n' + ' },\n' + '\n' + ' "prominent_elements": [\n' + ' {\n' + ' "name": "(String) Short label for the object (e.g., \'red sports car\', \'elderly man\')",\n' + ' "description": "(String) Detailed visual description of this specific element",\n' + ' "actions": [\n' + ' {\n' + ' "timestamp": "(String) Time range when this action occurs, e.g., \'[0.0s - 3.0s]\'",\n' + ' "action": "(String) Specific action description during this time period. **Direction must be described from the observer\'s perspective**. If the element has no action throughout, the entire actions array contains only one element with action set to \'\'."\n' + ' }\n' + ' ],\n' + ' "location": "(String) Precise position in the frame or main area of activity (from observer\'s perspective)",\n' + ' "relative_size": "(String) small / medium / large / dominant",\n' + ' "shape_and_color": "(String) Basic geometric shape and dominant colors",\n' + ' "texture": "(String) e.g., smooth, rough, metallic, furry, glossy, matte",\n' + ' "appearance_details": "(String) Specific details such as patterns, text physically printed on the object, wear marks, or distinctive markings",\n' + ' "relationship": "(String) This object\'s spatial or contextual relationship with other elements in the scene. If this object obscures text, specify exactly what it covers (e.g., \'blocking the letter O in COW). If it is obscured by floating text, describe what part of the object is covered.",\n' + ' "orientation": "(String) e.g., upright, tilted, horizontal, facing away, diagonal",\n' + ' \n' + ' "pose": "(String) Body posture and its changes (human/humanoid only, otherwise empty)",\n' + ' "expression": "(String) Facial expression and emotional changes (human/humanoid only, otherwise empty)",\n' + ' "clothing": "(String) Clothing description, including colors and styles (human/humanoid only, otherwise empty)",\n' + ' "gender": "(String) Apparent gender (human/humanoid only, otherwise empty)",\n' + ' "skin_tone_and_texture": "(String) Skin appearance (human/humanoid only, otherwise empty)",\n' + ' \n' + ' "is_cluster": "true (only for cluster objects, omit otherwise)",\n' + ' "number_of_objects": "(String) Exact number if countable, otherwise \'several\' (3-6), \'many\' (7-20), or \'numerous\' (20+)"\n' + ' }\n' + ' ],\n' + '\n' + ' "camera_info": {\n' + ' "color": "(String) Warm, Cool, Mixed, Saturated, Desaturated, Black and White, Red, Orange, Yellow, Green, Cyan, Blue, Magenta, or Pink",\n' + ' "frame_size": "(String) Extreme Wide, Wide, Medium Wide, Medium, Medium Close Up, Close Up, or Extreme Close Up",\n' + ' "shot_type_angle": "(String) High angle, Low angle, Dutch angle, Overhead, Aerial, or Eye level",\n' + ' "lens_size": "(String) Ultra Wide / Fisheye, Wide, Medium, Long Lens, or Telephoto",\n' + ' "composition": "(String) Center, Balanced, Symmetrical, Left heavy, Right heavy, or Short side",\n' + ' "lighting": "(String) Hard light, Soft light, High contrast, Low contrast, Side light, Top light, Underlight, Backlight, Edge light, or Silhouette",\n' + ' "lighting_type": "(String) Daylight, Sunny, Overcast, Moonlight, Artificial light, Practical light, Tungsten, Fluorescent, Firelight, or Mixed light"\n' + ' }\n' + '}\n' + '```\n' + '\n' + '## IMPORTANT RULES\n' + '\n' + '1. **Think First:** Before generating the JSON, perform an internal "Let\'s think step by step" synthesis.\n' + '2. **Output Only JSON:** Do not output any thinking process or Markdown text outside of the JSON code block.\n' + '3. **Strict Fidelity (No Modification)**:\n' + '- Identity: Never alter the identity of subjects (e.g., a "real tiger" remains "real tiger").\n' + '- Actions: Verbs must be preserved exactly (e.g., "walking" cannot be changed to "running"). Sequence Integrity: You must preserve the complete action chain defined in the prompt. Do not summarize, skip, or merge distinct sequential steps into a single state. No Collective Summarization: Do not use summary phrases (e.g., "a sequence of...", "various movements") to cover multiple steps. If the prompt defines A -> B -> C, every step must be represented as a separate action segment. Any skipped step or summarized sequence is a CRITICAL ERROR.\n' + '- Quantity: If a number is specified, it must be exact. Quantifiers must be strictly preserved. "All" means all, "every" means every, "a single" means exactly 1. Do not paraphrase quantifiers into vague terms.\n' + '- Spatial Integrity: Relative positions (e.g., "A on the left of B") must be fixed as stated.\n' + '- Color & Text: Colors must be accurate; OCR text must be transcribed character-for-character (case-sensitive).\n' + '4. **Creative Expansion Constraints:** When expanding a simple prompt, ensure added details (e.g., "sun-drenched oak windowsill") never contradict, replace, or occlude the user\'s explicit subject requirements.\n' + '5. **Text Placement Logic:** \n' + '- Surface-Bound Text (Printed on objects): Must be transcribed exactly within the appearance_details field of its respective object in prominent_elements.\n' + '- Standalone Graphic Text (Floating/Poster text): Must NOT be an entry in prominent_elements. Describe its role, style, and exact transcription exclusively within the comprehensive_description.\n' + "- Dynamic Text: If the scene contains moving or updating text (e.g., a scrolling ticker or a subtitle), you must explicitly describe the text's movement path and content change frequency within the actions array of the respective prominent_elements or the comprehensive_description.\n" + '6. **Occlusion & Relational Mapping:** Explicitly document overlaps. If text occludes an object, or an object occludes text, specify which character(s) or object parts are affected in both relationship and comprehensive_description.\n' + '7. **Handle Missing Data:** If an attribute is not applicable to an element (e.g., pose for a mountain), set its value to "" (an empty string). Never use "N/A", "unknown", "not applicable", etc.\n' + '8. **Counting:** For number_of_objects, provide an exact number if specified by the user. Otherwise, use: "several" (3-6), "many" (7-20), or "numerous" (20+).\n' + '9. **Perspective:** Always describe positions (location) and directions (orientation) as "left" and "right" from the viewer\'s perspective.\n' + '10. **Visual Descriptive Realism:** Avoid abstract or emotional adjectives (e.g., "beautiful," "sad"). Instead, translate them into visually observable details (e.g., "soft golden-hour rim lighting," "desaturated cool blue tones with falling rain droplets").\n' + '11. **Clusters:** When describing a cluster (e.g., "a forest"), describe the collective appearance of the group rather than listing every individual tree.\n' + '12. **Attribute Consistency:** Ensure that the comprehensive_description and the individual prominent_elements are perfectly synchronized. Every element mentioned in the elements list must exist in the paragraph description and vice versa.\n' + '13. **Negative Constraints:** Any explicit prohibitions (e.g., "no text," "no background") must be strictly upheld. Never add elements that the user has explicitly requested to exclude.\n' + '14. **Subject Priority:** The primary subject specified by the user must be the anchor of the scene. Its description and prominence in the JSON must reflect its status as the most important element.\n' + '15. **Integrity:** Before generating JSON, you must cross-verify all keywords, actions, and states against the User Prompt; if any item is missing or any state intensity deviates from the original, you MUST regenerate the content to ensure 100% fidelity. \n' + '16. **Non-OCR Classification:** Do not force objects into "text" status. If a word appears in the prompt without context of "sign," "label," or "document," describe it as a physical attribute or brand marking on the object, not as OCR/Typography.\n' + "17. **Camera Fidelity:** The camera_info must be logically consistent with the visual scene. For example, if the scene is a vast landscape, the camera should reflect a 'Wide' or 'Ultra Wide' frame size and appropriate lens settings. Avoid contradictory pairings (e.g., 'Extreme Close Up' with a 'Wide' lens).\n" + '18. **Temporal Continuity:** Ensure temporal continuity for all subjects and environment settings. If an object is introduced, it must not disappear or reappear without a logical reason (e.g., leaving the frame or being occluded). Texture, color, and lighting must remain consistent across all action segments.\n' + "19. **Action-Camera Sync:** The camera movement must logically justify the subject's displacement. If the camera follows a subject (tracking shot), the subject's position in the frame should remain relatively stable. If the camera is static, the subject must show movement within the frame.\n" + "20. **Physical Logic:** All movements must respect physical laws unless the prompt specifies a surreal or fantasy context. Avoid 'gliding' motions; ensure footsteps or object interactions (e.g., picking up an item) align with the timestamped actions.\n" + '21. **Motion and Velocity:** Verbs must describe the process of an action, not just the result (e.g., use \'rising from the chair\' instead of just \'standing\'). Every action in the actions array must include a velocity modifier (e.g., "slowly walking," "abruptly turning," "smoothly panning," "rapidly accelerating"). Avoid generic verbs without speed or force descriptors.\n' + '22. **Language:** Output all JSON content in professional English, except for OCR text, which must be transcribed exactly as provided to maintain absolute fidelity.\n' + '23. **Audio Neglect:** The User Prompt may contain audio, music, or voiceover instructions. Strictly ignore all audio-related instructions. Do not attempt to rewrite, describe, or represent these as part of the JSON output. Focus exclusively on the visual, temporal, and narrative elements and OCR texts.\n' + '24. **Duration Adherence:** A target video duration is provided in the input. Every timestamp in the `actions` array MUST fall within `[0.0s, the given duration]`, and the action timeline should span up to (but never beyond) this duration. Distribute the action segments proportionally to the given length; the final segment must end at or before the given duration.\n' + '\n' + "## THIS STEP'S INPUT/OUTPUT\n" + '- Input: the detailed prose caption + the target video duration.\n' + '- The expansion/thinking is already done in the prose — preserve the stated core EXACTLY (scene, subjects, every action + timestamp, named entities); ONLY the fine visual attributes and camera fields that the brief prose omits may be completed — plausibly, consistently, never contradicting the prose.\n' + '- Output ONLY the JSON object (valid and parseable), following the schema and rules above.' +) + +IMAGE_STEP1_EXPAND = ( + 'You write ONE short, natural, standalone IMAGE caption in English from a brief user image\n' + 'prompt — as if briefly describing a real photo of that idea.\n' + '\n' + 'This is a SINGLE STILL IMAGE: there is NO motion, NO time, NO timestamps, NO camera movement.\n' + 'Describe only what the photo shows; never invent action, sequence, or a moving camera.\n' + '\n' + 'HARD LENGTH LIMIT: the output MUST be UNDER 1000 characters. Aim for roughly 400-800\n' + 'characters. Concise is a top priority.\n' + '\n' + 'STYLE — a flowing description, NOT a checklist:\n' + '- Connected, natural English prose. NO headings, NO bullets, NO field labels.\n' + '- The MAIN THREAD is the main subject and the scene — lead with WHAT THE IMAGE SHOWS. Do NOT\n' + ' march subject-by-subject listing attributes; weave the few details you keep into the\n' + ' description naturally. One cohesive paragraph; never pad to add length.\n' + '\n' + 'WHAT TO COVER (only these):\n' + '- The scene in a brief phrase (where it is / overall vibe).\n' + '- Each main subject by name/what-it-is plus ONE most defining visual trait — not a full\n' + ' appearance list. The key spatial arrangement only if it defines the image.\n' + '- A one-word shot type only if it matters (e.g. close-up, wide shot, aerial); otherwise skip the camera.\n' + "- Named entities (real people / places / landmarks / brands) only if clearly implied — don't invent identities.\n" + '\n' + 'FAITHFUL EXPANSION:\n' + '- Stay consistent with the prompt; elaborate plausibly but never contradict it. Commit to one\n' + " coherent interpretation; don't hedge with alternatives.\n" + '- Keep subject identities and counts consistent.\n' + '\n' + 'Output ONLY the caption text — no preamble, no headings, no explanation.\n' + '\n' + '\n' + "## EXAMPLES — one representative example per type. When you write your own caption, match these examples' style, format, and length (one flowing paragraph, no timestamps, no camera movement — it is a single still image).\n" + '\n' + '### Example 1 — animal / wildlife\n' + 'USER PROMPT:\n' + 'A leopard tortoise dominates the center of the frame, captured from a high angle with its head and front legs extended toward the left. Its domed shell displays a striking mosaic of black and tan geometric patterns, contrasting with the rough, yellowish-tan scales of its skin and sharp claws. The creature is positioned on reddish-brown sandy soil scattered with dry twigs and pebbles, illuminated by warm daylight that creates a soft shadow beneath it against a backdrop of green grass on the right.\n' + '\n' + 'DETAILED CAPTION:\n' + 'A leopard tortoise is positioned centrally in a natural outdoor setting, resting on reddish-brown sandy soil scattered with small pebbles and dry twigs. Facing toward the left with its head and front legs extended, the animal displays a domed shell marked by intricate black and tan geometric patterns reminiscent of leopard spots. Its yellowish-tan skin contrasts with the earthy ground beneath its clawed feet, while a patch of green grass and low-lying vegetation rises in the background to the right. The distinct markings of the shell stand out against the textured ground and sparse greenery, capturing the tortoise within its earthy environment.\n' + '\n' + '### Example 2 — architecture / landmark\n' + 'USER PROMPT:\n' + '极度广角镜头，高角度拍摄。明亮日光，硬光，暖色调，清晰阴影。宏伟的古典风格宫殿庭院，浅色石材。中央巨大的多层拱形入口，饰有金色和蓝色装饰带，深色凹陷内部可见小金门。宽阔石阶通向圆形分层平台。两侧对称亭阁，顶部为反射金色圆顶，蓝色瓷砖拱门，装饰攀爬绿藤和粉色花朵。露台布满茂盛绿植和小花。若干穿着传统长袍（白、灰、棕、红）的小人物散布庭院以示比例。背景岩石山坡，远处建筑，晴朗明亮天空。写实建筑摄影。\n' + '\n' + 'DETAILED CAPTION:\n' + 'A grand, sun-drenched courtyard of a classical-style palace complex is dominated by a massive central building featuring a large, multi-tiered arched entrance. Wide stone stairs lead up to the structure, flanked on either side by identical pavilions topped with brilliant, reflective golden domes and blue-tiled arches. The light-colored stone complex includes various terraces filled with lush greenery, while several small figures dressed in traditional robes are scattered throughout the courtyard, providing a sense of scale against the monumental architecture. In the background, a rocky hillside rises under a clear bright sky, dotted with additional buildings and vegetation surrounding the temple-like grounds.\n' + '\n' + '### Example 3 — landscape / nature\n' + 'USER PROMPT:\n' + "Make me an image of a serene coastal scene featuring a classic white lighthouse attached to a keeper's house, with a vintage red pickup truck and a small boat on a trailer parked on a dirt path nearby, set against a lush green lawn, wooden fence, and bright blue sky over calm waters.\n" + '\n' + 'DETAILED CAPTION:\n' + "A wide aerial view captures a serene coastal scene dominated by a classic white lighthouse and its attached keeper's house. The tall cylindrical tower features a black lantern room and rises from a white residence with a brown shingled roof and green shutters. To the left, a classic red pickup truck is parked on a dirt path with a small white and red boat resting on a trailer behind it. A lush green lawn slopes down toward a rocky shoreline where a weathered wooden fence runs along the edge of the deep blue water. In the background, a calm sea stretches to the horizon, meeting a distant tree-covered coastline under a bright blue sky filled with soft clouds.\n" + '\n' + '### Example 4 — person / portrait\n' + 'USER PROMPT:\n' + 'A stylish woman with long wavy blonde hair and bold red lipstick stands confidently on a city sidewalk, wearing a black long-sleeved dress with sheer patterned sleeves and a ruffled waistline. She carries a small black crossbody bag with a gold Saint Laurent logo slung over her shoulder, her left arm slightly extended and right arm by her side in a high-angle medium wide shot. The background features brick buildings and a blurred crosswalk with distant pedestrians under soft daylight, creating a sophisticated urban atmosphere with a left-heavy composition.\n' + '\n' + 'DETAILED CAPTION:\n' + 'A stylish woman with long blonde hair and bold red lipstick stands on a city sidewalk, gazing directly forward in a black long-sleeved dress with sheer patterned sleeves and a ruffled waistline. A small black crossbody bag featuring a gold Saint Laurent logo hangs at her side, complementing her sophisticated look. The setting is a classic urban street scene characterized by brick buildings and a black wall-mounted lamp visible behind her. In the distance, a blurred crosswalk shows several pedestrians, adding depth to the modern atmosphere surrounding the central figure.' +) + +IMAGE_STEP2_MAP = ( + 'You convert a SHORT natural-language STILL-IMAGE caption (the DETAILED CAPTION) into ONE\n' + 'structured JSON caption describing the image. Output ONLY the JSON object — no prose, no code fence.\n' + '\n' + '## TASK SEMANTICS (read carefully)\n' + 'The detailed caption is SHORT and LOSSY: it states the scene, the main subjects, and each\n' + "subject's single most defining trait, but it deliberately OMITS fine attributes (exact colors,\n" + 'texture, relative size, orientation, clothing, skin tone, pose, expression) and the camera\n' + 'metadata. Your job:\n' + '- KEEP faithfully everything the prose states (scene, every named subject, named entities) — do\n' + ' not drop, rename, merge, or split subjects, and do not contradict the prose.\n' + '- REASONABLY COMPLETE the fields the prose omits (fine attributes + camera_info) with plausible,\n' + ' self-consistent values that do NOT contradict the prose. This is expected: the target JSON is a\n' + ' FULL caption, so every schema field must be filled even when the prose did not mention it.\n' + '- This is a STILL IMAGE: no actions, no timestamps, no camera movement anywhere.\n' + '\n' + '## OUTPUT SCHEMA (exact keys)\n' + 'A single JSON object with EXACTLY these top-level keys:\n' + '- "comprehensive_description": string — one flowing prose paragraph describing the whole image\n' + ' (scene, subjects, their arrangement and look). No field labels inside it.\n' + '- "camera_info": object with EXACTLY these 7 string keys, each set to ONE value from its list:\n' + ' - "color": Warm | Cool | Mixed | Saturated | Desaturated | White | Green | Blue | Red | Cyan\n' + ' - "frame_size": Extreme Close Up | Close Up | Medium Close Up | Medium | Medium Wide | Wide | Extreme Wide\n' + ' - "shot_type_angle": Eye level | Low angle | High angle | Overhead | Aerial\n' + ' - "lens_size": Ultra Wide / Fisheye | Wide | Medium | Long Lens | Telephoto\n' + ' - "composition": Center | Balanced | Symmetrical | Left heavy | Right heavy\n' + ' - "lighting": Soft light | Hard light | Top light | Underlight | Backlight\n' + ' - "lighting_type": Daylight | Artificial light\n' + '- "world_knowledge": array of strings — named entities / real-world facts (people, places,\n' + ' landmarks, brands) stated or clearly implied by the prose; [] if none. NEVER invent identities.\n' + '- "prominent_elements": array of objects, one per notable subject/object. Each element has:\n' + ' - always: "name", "description", "location", "relative_size" (one of: dominant | large | medium | small),\n' + ' "shape_and_color", "texture", "appearance_details", "relationship", "orientation"\n' + ' - for a PERSON, additionally: "pose", "expression", "clothing", "gender" (male | female),\n' + ' "skin_tone_and_texture"\n' + ' - for a GROUP/crowd of like items, additionally: "is_cluster": true and "number_of_objects": ""\n' + '\n' + '## RULES\n' + '- Output STRICT valid JSON only (double quotes, no trailing commas, no comments, no code fence).\n' + '- Fill EVERY field; never leave a required field empty. Values you complete must be consistent\n' + ' with the prose and with each other.\n' + '- Keep the number and identity of prominent_elements aligned with the subjects the prose names\n' + ' (you may add a clearly-implied background element, but do not invent unrelated subjects).\n' + '- Copy named entities verbatim into world_knowledge.\n' + '- No actions, no timestamps, no camera-movement fields — this is a still image.' +) diff --git a/examples/lingbot_video/model_training/special/split_training/lingbot-video-dense-1.3b_t2v.sh b/examples/lingbot_video/model_training/special/split_training/lingbot-video-dense-1.3b_t2v.sh new file mode 100644 index 0000000000000000000000000000000000000000..22b66f1d5f4880b5e5f9bbcb0d2d2ee2d95e097a --- /dev/null +++ b/examples/lingbot_video/model_training/special/split_training/lingbot-video-dense-1.3b_t2v.sh @@ -0,0 +1,44 @@ +modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "lingbot_video/lingbot-video-dense-1.3b_t2v/*" --local_dir ./data/diffsynth_example_dataset + +# Stage 1: cache deterministic preprocessing outputs. +accelerate launch examples/lingbot_video/model_training/train.py \ + --dataset_base_path data/diffsynth_example_dataset/lingbot_video/lingbot-video-dense-1.3b_t2v \ + --dataset_metadata_path data/diffsynth_example_dataset/lingbot_video/lingbot-video-dense-1.3b_t2v/metadata.json \ + --data_file_keys video \ + --height 480 \ + --width 832 \ + --num_frames 81 \ + --dataset_repeat 1 \ + --model_id_with_origin_paths 'Robbyant/lingbot-video-dense-1.3b:transformer/diffusion_pytorch_model.safetensors,Qwen/Qwen3-VL-4B-Instruct:*.safetensors,Robbyant/lingbot-video-dense-1.3b:vae/diffusion_pytorch_model.safetensors' \ + --processor_path Qwen/Qwen3-VL-4B-Instruct: \ + --learning_rate 1e-4 \ + --num_epochs 5 \ + --remove_prefix_in_ckpt pipe.dit. \ + --output_path ./models/train/lingbot-video-dense-1.3b_t2v_split_cache \ + --lora_base_model dit \ + --lora_target_modules to_q,to_k,to_v,to_out \ + --lora_rank 32 \ + --use_gradient_checkpointing \ + --offload_models Robbyant/lingbot-video-dense-1.3b:transformer/diffusion_pytorch_model.safetensors \ + --task sft:data_process + +# Stage 2: train LoRA from the cached dataset. +accelerate launch examples/lingbot_video/model_training/train.py \ + --dataset_base_path ./models/train/lingbot-video-dense-1.3b_t2v_split_cache \ + --data_file_keys video \ + --height 480 \ + --width 832 \ + --num_frames 81 \ + --dataset_repeat 50 \ + --model_id_with_origin_paths 'Robbyant/lingbot-video-dense-1.3b:transformer/diffusion_pytorch_model.safetensors,Qwen/Qwen3-VL-4B-Instruct:*.safetensors,Robbyant/lingbot-video-dense-1.3b:vae/diffusion_pytorch_model.safetensors' \ + --processor_path Qwen/Qwen3-VL-4B-Instruct: \ + --learning_rate 1e-4 \ + --num_epochs 5 \ + --remove_prefix_in_ckpt pipe.dit. \ + --output_path ./models/train/lingbot-video-dense-1.3b_t2v_split \ + --lora_base_model dit \ + --lora_target_modules to_q,to_k,to_v,to_out \ + --lora_rank 32 \ + --use_gradient_checkpointing \ + --offload_models 'Qwen/Qwen3-VL-4B-Instruct:*.safetensors,Robbyant/lingbot-video-dense-1.3b:vae/diffusion_pytorch_model.safetensors' \ + --task sft:train diff --git a/examples/lingbot_video/model_training/special/split_training/validate.py b/examples/lingbot_video/model_training/special/split_training/validate.py new file mode 100644 index 0000000000000000000000000000000000000000..885163fa97bd4bc26a1ed2d41119d597f1a19515 --- /dev/null +++ b/examples/lingbot_video/model_training/special/split_training/validate.py @@ -0,0 +1,34 @@ +import torch +import json +from diffsynth.utils.data import save_video +from diffsynth.pipelines.lingbot_video import LingBotVideoPipeline, ModelConfig +from modelscope import dataset_snapshot_download + + +pipe = LingBotVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="Robbyant/lingbot-video-dense-1.3b", origin_file_pattern="transformer/diffusion_pytorch_model.safetensors"), + ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern="*.safetensors"), + ModelConfig(model_id="Robbyant/lingbot-video-dense-1.3b", origin_file_pattern="vae/diffusion_pytorch_model.safetensors"), + ], + processor_config=ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern=""), +) +pipe.load_lora(pipe.dit, './models/train/lingbot-video-dense-1.3b_t2v_split/epoch-4.safetensors', alpha=1) +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/diffsynth_example_dataset", + local_dir="data/diffsynth_example_dataset", + allow_file_pattern="lingbot_video/lingbot-video-dense-1.3b_t2v/*", +) +with open("data/diffsynth_example_dataset/lingbot_video/lingbot-video-dense-1.3b_t2v/t2v_example_1.json", "r", encoding="utf-8") as f: + caption = json.load(f) + +video = pipe( + prompt=caption, + negative_prompt=pipe.default_negative_prompt, + height=480, width=832, num_frames=81, + num_inference_steps=40, cfg_scale=3.0, + seed=0, +) +save_video(video, 'split_training_lingbot-video-dense-1.3b_t2v.mp4', fps=15, quality=10) diff --git a/examples/lingbot_video/model_training/train.py b/examples/lingbot_video/model_training/train.py new file mode 100644 index 0000000000000000000000000000000000000000..92afb7c4438f696118d5f37954e0819da4a66806 --- /dev/null +++ b/examples/lingbot_video/model_training/train.py @@ -0,0 +1,182 @@ +import torch, os, argparse, accelerate +from diffsynth.core import UnifiedDataset +from diffsynth.pipelines.lingbot_video import LingBotVideoPipeline +from diffsynth.diffusion import * +os.environ["TOKENIZERS_PARALLELISM"] = "false" + + +class LingBotVideoTrainingModule(DiffusionTrainingModule): + def __init__( + self, + model_paths=None, model_id_with_origin_paths=None, + processor_path=None, + trainable_models=None, + lora_base_model=None, lora_target_modules="", lora_rank=32, lora_checkpoint=None, + preset_lora_path=None, preset_lora_model=None, + use_gradient_checkpointing=True, + use_gradient_checkpointing_offload=False, + first_frame_as_condition=False, + extra_inputs=None, + fp8_models=None, + offload_models=None, + quant_options=None, + resume_from_checkpoint=None, remove_prefix_in_ckpt=None, + device="cpu", + task="sft", + max_timestep_boundary=1.0, + min_timestep_boundary=0.0, + ): + super().__init__() + # Load models. The Qwen3-VL processor (tokenizer + image/video processor) is + # passed separately via `processor_config`, mirroring the inference pipeline. + model_configs = self.parse_model_configs(model_paths, model_id_with_origin_paths, fp8_models=fp8_models, offload_models=offload_models, quant_options=quant_options, device=device) + processor_config = self.parse_path_or_model_id(processor_path) + self.pipe = LingBotVideoPipeline.from_pretrained(torch_dtype=torch.bfloat16, device=device, model_configs=model_configs, processor_config=processor_config) + self.pipe = self.split_pipeline_units(task, self.pipe, trainable_models, lora_base_model) + self.resume_from_checkpoint(resume_from_checkpoint, remove_prefix_in_ckpt) + + # Training mode: FlowMatchScheduler (Wan template) runs the full 1000-step + # schedule and the SFT loss samples a random timestep. Attention-only LoRA is the + # default; pass --lora_target_modules "to_q,to_k,to_v,to_out" to skip MoE/router. + self.switch_pipe_to_training_mode( + self.pipe, trainable_models, + lora_base_model, lora_target_modules, lora_rank, lora_checkpoint, + preset_lora_path, preset_lora_model, + task=task, + ) + + # Store other configs + self.use_gradient_checkpointing = use_gradient_checkpointing + self.use_gradient_checkpointing_offload = use_gradient_checkpointing_offload + self.first_frame_as_condition = first_frame_as_condition + self.extra_inputs = extra_inputs.split(",") if extra_inputs is not None else [] + self.fp8_models = fp8_models + self.task = task + self.task_to_loss = { + "sft:data_process": lambda pipe, *args: args, + "sft": lambda pipe, inputs_shared, inputs_posi, inputs_nega: FlowMatchSFTLoss(pipe, **inputs_shared, **inputs_posi), + "sft:train": lambda pipe, inputs_shared, inputs_posi, inputs_nega: FlowMatchSFTLoss(pipe, **inputs_shared, **inputs_posi), + } + self.max_timestep_boundary = max_timestep_boundary + self.min_timestep_boundary = min_timestep_boundary + + def get_pipeline_inputs(self, data): + # The metadata "prompt" column stores the serialised structured-JSON caption + # (see rewrite_captions.py); the pipeline's PromptEmbedder consumes it as-is. + inputs_posi = {"prompt": data["prompt"]} + inputs_nega = {} + inputs_shared = { + # Assume you are using this pipeline for inference, + # please fill in the input parameters. + "input_video": data["video"], + "height": data["video"][0].size[1], + "width": data["video"][0].size[0], + "num_frames": len(data["video"]), + # Please do not modify the following parameters + # unless you clearly know what this will cause. + "cfg_scale": 1, + "seed": None, + "rand_device": self.pipe.device, + "use_gradient_checkpointing": self.use_gradient_checkpointing, + "use_gradient_checkpointing_offload": self.use_gradient_checkpointing_offload, + "max_timestep_boundary": self.max_timestep_boundary, + "min_timestep_boundary": self.min_timestep_boundary, + } + if self.first_frame_as_condition: + # Image-to-video (TI2V) LoRA: condition on the clip's own first frame. The + # ImageEmbedder unit VAE-encodes it to first_frame_latents and feeds it to the + # text encoder as vlm_image; FlowMatchSFTLoss then pins that clean latent into + # the first temporal slot and excludes it from the loss (see diffsynth/diffusion/ + # loss.py). No separate condition column or core change is needed. If your + # dataset instead ships a distinct condition frame, drop this flag and pass it via + # --extra_inputs input_image (with the column added to --data_file_keys). + inputs_shared["input_image"] = data["video"][0] + inputs_shared = self.parse_extra_inputs(data, self.extra_inputs, inputs_shared) + return inputs_shared, inputs_posi, inputs_nega + + def forward(self, data, inputs=None): + if inputs is None: inputs = self.get_pipeline_inputs(data) + inputs = self.transfer_data_to_device(inputs, self.pipe.device, self.pipe.torch_dtype) + for unit in self.pipe.units: + inputs = self.pipe.unit_runner(unit, self.pipe, *inputs) + loss = self.task_to_loss[self.task](self.pipe, *inputs) + return loss + + +def lingbot_video_parser(): + parser = argparse.ArgumentParser(description="Simple example of a LingBot-Video training script.") + parser = add_general_config(parser) + parser = add_video_size_config(parser) + parser.add_argument("--processor_path", type=str, default=None, help="Path to the Qwen3-VL processor directory (or `model_id:origin_file_pattern`). Used to tokenize prompts.") + parser.add_argument("--first_frame_as_condition", default=False, action="store_true", help="Image-to-video (TI2V) LoRA: condition on each clip's own first frame (pinned as a clean latent and excluded from the loss).") + parser.add_argument("--max_timestep_boundary", type=float, default=1.0, help="Max timestep boundary (fraction of the training schedule, in [0, 1]).") + parser.add_argument("--min_timestep_boundary", type=float, default=0.0, help="Min timestep boundary (fraction of the training schedule, in [0, 1]).") + parser.add_argument("--initialize_model_on_cpu", default=False, action="store_true", help="Whether to initialize models on CPU.") + return parser + + +if __name__ == "__main__": + parser = lingbot_video_parser() + args = parser.parse_args() + accelerator = accelerate.Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + kwargs_handlers=[accelerate.DistributedDataParallelKwargs(find_unused_parameters=args.find_unused_parameters)], + ) + dataset = UnifiedDataset( + base_path=args.dataset_base_path, + metadata_path=args.dataset_metadata_path, + repeat=args.dataset_repeat, + data_file_keys=args.data_file_keys.split(","), + main_data_operator=UnifiedDataset.default_video_operator( + base_path=args.dataset_base_path, + max_pixels=args.max_pixels, + height=args.height, + width=args.width, + height_division_factor=16, + width_division_factor=16, + num_frames=args.num_frames, + time_division_factor=4, + time_division_remainder=1, + ), + ) + model = LingBotVideoTrainingModule( + model_paths=args.model_paths, + model_id_with_origin_paths=args.model_id_with_origin_paths, + processor_path=args.processor_path, + trainable_models=args.trainable_models, + lora_base_model=args.lora_base_model, + lora_target_modules=args.lora_target_modules, + lora_rank=args.lora_rank, + lora_checkpoint=args.lora_checkpoint, + preset_lora_path=args.preset_lora_path, + preset_lora_model=args.preset_lora_model, + use_gradient_checkpointing=args.use_gradient_checkpointing, + use_gradient_checkpointing_offload=args.use_gradient_checkpointing_offload, + first_frame_as_condition=args.first_frame_as_condition, + extra_inputs=args.extra_inputs, + fp8_models=args.fp8_models, + offload_models=args.offload_models, + quant_options=args.quant_options, + resume_from_checkpoint=args.resume_from_checkpoint, + remove_prefix_in_ckpt=args.remove_prefix_in_ckpt, + task=args.task, + device="cpu" if (args.initialize_model_on_cpu or args.enable_model_cpu_offload) else accelerator.device, + max_timestep_boundary=args.max_timestep_boundary, + min_timestep_boundary=args.min_timestep_boundary, + ) + model_logger = ModelLogger( + args.output_path, + remove_prefix_in_ckpt=args.remove_prefix_in_ckpt, + enable_tensorboard_log=args.enable_tensorboard_log, + enable_swanlab_log=args.enable_swanlab_log, + swanlab_project=args.swanlab_project, + enable_wandb_log=args.enable_wandb_log, + wandb_project=args.wandb_project, + enable_csv_log=args.enable_csv_log, + ) + launcher_map = { + "sft:data_process": launch_data_process_task, + "sft": launch_training_task, + "sft:train": launch_training_task, + } + launcher_map[args.task](accelerator, dataset, model, model_logger, args=args) diff --git a/examples/lingbot_video/model_training/validate_full/lingbot-video-dense-1.3b_t2v.py b/examples/lingbot_video/model_training/validate_full/lingbot-video-dense-1.3b_t2v.py new file mode 100644 index 0000000000000000000000000000000000000000..e2ef35b1fac9f5c456282663fb80e48cd0b7330b --- /dev/null +++ b/examples/lingbot_video/model_training/validate_full/lingbot-video-dense-1.3b_t2v.py @@ -0,0 +1,36 @@ +import torch +import json +from diffsynth.utils.data import save_video +from diffsynth.pipelines.lingbot_video import LingBotVideoPipeline, ModelConfig +from diffsynth import load_state_dict +from modelscope import dataset_snapshot_download + + +pipe = LingBotVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="Robbyant/lingbot-video-dense-1.3b", origin_file_pattern="transformer/diffusion_pytorch_model.safetensors"), + ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern="*.safetensors"), + ModelConfig(model_id="Robbyant/lingbot-video-dense-1.3b", origin_file_pattern="vae/diffusion_pytorch_model.safetensors"), + ], + processor_config=ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern=""), +) +state_dict = load_state_dict("models/train/lingbot-video-dense-1.3b_t2v_full/epoch-1.safetensors") +pipe.dit.load_state_dict(state_dict) +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/diffsynth_example_dataset", + local_dir="data/diffsynth_example_dataset", + allow_file_pattern="lingbot_video/lingbot-video-dense-1.3b_t2v/*", +) +with open("data/diffsynth_example_dataset/lingbot_video/lingbot-video-dense-1.3b_t2v/t2v_example_1.json", "r", encoding="utf-8") as f: + caption = json.load(f) + +video = pipe( + prompt=caption, + negative_prompt=pipe.default_negative_prompt, + height=480, width=832, num_frames=81, + num_inference_steps=40, cfg_scale=3.0, + seed=0, +) +save_video(video, "video_lingbot-video-dense-1.3b_t2v.mp4", fps=15, quality=10) diff --git a/examples/lingbot_video/model_training/validate_full/lingbot-video-dense-1.3b_ti2v.py b/examples/lingbot_video/model_training/validate_full/lingbot-video-dense-1.3b_ti2v.py new file mode 100644 index 0000000000000000000000000000000000000000..c1dda2ff57ce02fd3dd9fdb3f00950272cb51ba6 --- /dev/null +++ b/examples/lingbot_video/model_training/validate_full/lingbot-video-dense-1.3b_ti2v.py @@ -0,0 +1,42 @@ +import os +import json +import torch +from PIL import Image +from diffsynth.utils.data import save_video +from diffsynth.pipelines.lingbot_video import LingBotVideoPipeline, ModelConfig +from diffsynth import load_state_dict +from modelscope import dataset_snapshot_download + +pipe = LingBotVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="Robbyant/lingbot-video-dense-1.3b", origin_file_pattern="transformer/diffusion_pytorch_model.safetensors"), + ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern="*.safetensors"), + ModelConfig(model_id="Robbyant/lingbot-video-dense-1.3b", origin_file_pattern="vae/diffusion_pytorch_model.safetensors"), + ], + processor_config=ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern=""), +) +state_dict = load_state_dict("models/train/lingbot-video-dense-1.3b_ti2v_full/epoch-1.safetensors") +pipe.dit.load_state_dict(state_dict) + +# The condition first frame and its paired caption ship in the example dataset. +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/diffsynth_example_dataset", + local_dir="data/diffsynth_example_dataset", + allow_file_pattern="lingbot_video/lingbot-video-dense-1.3b_ti2v/*", +) +base = "data/diffsynth_example_dataset/lingbot_video/lingbot-video-dense-1.3b_ti2v" +with open(os.path.join(base, "ti2v_example.json"), "r", encoding="utf-8") as f: + caption = json.load(f) +input_image = Image.open(os.path.join(base, "ti2v_first_frame.png")).convert("RGB") + +video = pipe( + prompt=caption, + negative_prompt=pipe.default_negative_prompt, + input_image=input_image, + height=480, width=832, num_frames=81, + num_inference_steps=40, cfg_scale=3.0, + seed=0, +) +save_video(video, "video_lingbot-video-dense-1.3b_ti2v.mp4", fps=15, quality=10) diff --git a/examples/lingbot_video/model_training/validate_full/lingbot-video-moe-30b-a3b_t2v.py b/examples/lingbot_video/model_training/validate_full/lingbot-video-moe-30b-a3b_t2v.py new file mode 100644 index 0000000000000000000000000000000000000000..906bb0dcc889d7279ce84d3281f27ee74b34506a --- /dev/null +++ b/examples/lingbot_video/model_training/validate_full/lingbot-video-moe-30b-a3b_t2v.py @@ -0,0 +1,36 @@ +import torch +import json +from diffsynth.utils.data import save_video +from diffsynth.pipelines.lingbot_video import LingBotVideoPipeline, ModelConfig +from diffsynth import load_state_dict +from modelscope import dataset_snapshot_download + + +pipe = LingBotVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="transformer/diffusion_pytorch_model*.safetensors"), + ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern="*.safetensors"), + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="vae/diffusion_pytorch_model.safetensors"), + ], + processor_config=ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern=""), +) +state_dict = load_state_dict("models/train/lingbot-video-moe-30b-a3b_t2v_full/epoch-1.safetensors") +pipe.dit.load_state_dict(state_dict) +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/diffsynth_example_dataset", + local_dir="data/diffsynth_example_dataset", + allow_file_pattern="lingbot_video/lingbot-video-moe-30b-a3b_t2v/*", +) +with open("data/diffsynth_example_dataset/lingbot_video/lingbot-video-moe-30b-a3b_t2v/t2v_example_1.json", "r", encoding="utf-8") as f: + caption = json.load(f) + +video = pipe( + prompt=caption, + negative_prompt=pipe.default_negative_prompt, + height=480, width=832, num_frames=81, + num_inference_steps=40, cfg_scale=3.0, + seed=0, +) +save_video(video, "video_lingbot-video-moe-30b-a3b_t2v.mp4", fps=15, quality=10) diff --git a/examples/lingbot_video/model_training/validate_full/lingbot-video-moe-30b-a3b_ti2v.py b/examples/lingbot_video/model_training/validate_full/lingbot-video-moe-30b-a3b_ti2v.py new file mode 100644 index 0000000000000000000000000000000000000000..4093ae5ce1cde548cb3ddeb2f144928115f8e9f0 --- /dev/null +++ b/examples/lingbot_video/model_training/validate_full/lingbot-video-moe-30b-a3b_ti2v.py @@ -0,0 +1,42 @@ +import os +import json +import torch +from PIL import Image +from diffsynth.utils.data import save_video +from diffsynth.pipelines.lingbot_video import LingBotVideoPipeline, ModelConfig +from diffsynth import load_state_dict +from modelscope import dataset_snapshot_download + +pipe = LingBotVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="transformer/diffusion_pytorch_model*.safetensors"), + ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern="*.safetensors"), + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="vae/diffusion_pytorch_model.safetensors"), + ], + processor_config=ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern=""), +) +state_dict = load_state_dict("models/train/lingbot-video-moe-30b-a3b_ti2v_full/epoch-1.safetensors") +pipe.dit.load_state_dict(state_dict) + +# The condition first frame and its paired caption ship in the example dataset. +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/diffsynth_example_dataset", + local_dir="data/diffsynth_example_dataset", + allow_file_pattern="lingbot_video/lingbot-video-moe-30b-a3b_ti2v/*", +) +base = "data/diffsynth_example_dataset/lingbot_video/lingbot-video-moe-30b-a3b_ti2v" +with open(os.path.join(base, "ti2v_example.json"), "r", encoding="utf-8") as f: + caption = json.load(f) +input_image = Image.open(os.path.join(base, "ti2v_first_frame.png")).convert("RGB") + +video = pipe( + prompt=caption, + negative_prompt=pipe.default_negative_prompt, + input_image=input_image, + height=480, width=832, num_frames=81, + num_inference_steps=40, cfg_scale=3.0, + seed=0, +) +save_video(video, "video_lingbot-video-moe-30b-a3b_ti2v.mp4", fps=15, quality=10) diff --git a/examples/lingbot_video/model_training/validate_lora/lingbot-video-dense-1.3b_t2v.py b/examples/lingbot_video/model_training/validate_lora/lingbot-video-dense-1.3b_t2v.py new file mode 100644 index 0000000000000000000000000000000000000000..eae32cf06760b9c2c7c9d715f30c641d980d30ca --- /dev/null +++ b/examples/lingbot_video/model_training/validate_lora/lingbot-video-dense-1.3b_t2v.py @@ -0,0 +1,34 @@ +import torch +import json +from diffsynth.utils.data import save_video +from diffsynth.pipelines.lingbot_video import LingBotVideoPipeline, ModelConfig +from modelscope import dataset_snapshot_download + + +pipe = LingBotVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="Robbyant/lingbot-video-dense-1.3b", origin_file_pattern="transformer/diffusion_pytorch_model.safetensors"), + ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern="*.safetensors"), + ModelConfig(model_id="Robbyant/lingbot-video-dense-1.3b", origin_file_pattern="vae/diffusion_pytorch_model.safetensors"), + ], + processor_config=ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern=""), +) +pipe.load_lora(pipe.dit, "models/train/lingbot-video-dense-1.3b_t2v_lora/epoch-4.safetensors", alpha=1) +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/diffsynth_example_dataset", + local_dir="data/diffsynth_example_dataset", + allow_file_pattern="lingbot_video/lingbot-video-dense-1.3b_t2v/*", +) +with open("data/diffsynth_example_dataset/lingbot_video/lingbot-video-dense-1.3b_t2v/t2v_example_1.json", "r", encoding="utf-8") as f: + caption = json.load(f) + +video = pipe( + prompt=caption, + negative_prompt=pipe.default_negative_prompt, + height=480, width=832, num_frames=81, + num_inference_steps=40, cfg_scale=3.0, + seed=0, +) +save_video(video, "video_lingbot-video-dense-1.3b_t2v.mp4", fps=15, quality=10) diff --git a/examples/lingbot_video/model_training/validate_lora/lingbot-video-dense-1.3b_ti2v.py b/examples/lingbot_video/model_training/validate_lora/lingbot-video-dense-1.3b_ti2v.py new file mode 100644 index 0000000000000000000000000000000000000000..c694eda237472f9aab37676f9b81a788f6bf1fa1 --- /dev/null +++ b/examples/lingbot_video/model_training/validate_lora/lingbot-video-dense-1.3b_ti2v.py @@ -0,0 +1,40 @@ +import os +import json +import torch +from PIL import Image +from diffsynth.utils.data import save_video +from diffsynth.pipelines.lingbot_video import LingBotVideoPipeline, ModelConfig +from modelscope import dataset_snapshot_download + +pipe = LingBotVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="Robbyant/lingbot-video-dense-1.3b", origin_file_pattern="transformer/diffusion_pytorch_model.safetensors"), + ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern="*.safetensors"), + ModelConfig(model_id="Robbyant/lingbot-video-dense-1.3b", origin_file_pattern="vae/diffusion_pytorch_model.safetensors"), + ], + processor_config=ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern=""), +) +pipe.load_lora(pipe.dit, "models/train/lingbot-video-dense-1.3b_ti2v_lora/epoch-4.safetensors", alpha=1) + +# The condition first frame and its paired caption ship in the example dataset. +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/diffsynth_example_dataset", + local_dir="data/diffsynth_example_dataset", + allow_file_pattern="lingbot_video/lingbot-video-dense-1.3b_ti2v/*", +) +base = "data/diffsynth_example_dataset/lingbot_video/lingbot-video-dense-1.3b_ti2v" +with open(os.path.join(base, "ti2v_example.json"), "r", encoding="utf-8") as f: + caption = json.load(f) +input_image = Image.open(os.path.join(base, "ti2v_first_frame.png")).convert("RGB") + +video = pipe( + prompt=caption, + negative_prompt=pipe.default_negative_prompt, + input_image=input_image, + height=480, width=832, num_frames=81, + num_inference_steps=40, cfg_scale=3.0, + seed=0, +) +save_video(video, "video_lingbot-video-dense-1.3b_ti2v.mp4", fps=15, quality=10) diff --git a/examples/lingbot_video/model_training/validate_lora/lingbot-video-moe-30b-a3b_t2v.py b/examples/lingbot_video/model_training/validate_lora/lingbot-video-moe-30b-a3b_t2v.py new file mode 100644 index 0000000000000000000000000000000000000000..bfa78a60f5545cff9dda37d1fd53a4fe809c073c --- /dev/null +++ b/examples/lingbot_video/model_training/validate_lora/lingbot-video-moe-30b-a3b_t2v.py @@ -0,0 +1,34 @@ +import torch +import json +from diffsynth.utils.data import save_video +from diffsynth.pipelines.lingbot_video import LingBotVideoPipeline, ModelConfig +from modelscope import dataset_snapshot_download + + +pipe = LingBotVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="transformer/diffusion_pytorch_model*.safetensors"), + ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern="*.safetensors"), + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="vae/diffusion_pytorch_model.safetensors"), + ], + processor_config=ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern=""), +) +pipe.load_lora(pipe.dit, "models/train/lingbot-video-moe-30b-a3b_t2v_lora/epoch-4.safetensors", alpha=1) +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/diffsynth_example_dataset", + local_dir="data/diffsynth_example_dataset", + allow_file_pattern="lingbot_video/lingbot-video-moe-30b-a3b_t2v/*", +) +with open("data/diffsynth_example_dataset/lingbot_video/lingbot-video-moe-30b-a3b_t2v/t2v_example_1.json", "r", encoding="utf-8") as f: + caption = json.load(f) + +video = pipe( + prompt=caption, + negative_prompt=pipe.default_negative_prompt, + height=480, width=832, num_frames=81, + num_inference_steps=40, cfg_scale=3.0, + seed=0, +) +save_video(video, "video_lingbot-video-moe-30b-a3b_t2v.mp4", fps=15, quality=10) diff --git a/examples/lingbot_video/model_training/validate_lora/lingbot-video-moe-30b-a3b_ti2v.py b/examples/lingbot_video/model_training/validate_lora/lingbot-video-moe-30b-a3b_ti2v.py new file mode 100644 index 0000000000000000000000000000000000000000..2e3237f2679ee3bfe4d6b5e3f162971709525625 --- /dev/null +++ b/examples/lingbot_video/model_training/validate_lora/lingbot-video-moe-30b-a3b_ti2v.py @@ -0,0 +1,40 @@ +import os +import json +import torch +from PIL import Image +from diffsynth.utils.data import save_video +from diffsynth.pipelines.lingbot_video import LingBotVideoPipeline, ModelConfig +from modelscope import dataset_snapshot_download + +pipe = LingBotVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="transformer/diffusion_pytorch_model*.safetensors"), + ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern="*.safetensors"), + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="vae/diffusion_pytorch_model.safetensors"), + ], + processor_config=ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern=""), +) +pipe.load_lora(pipe.dit, "models/train/lingbot-video-moe-30b-a3b_ti2v_lora/epoch-4.safetensors", alpha=1) + +# The condition first frame and its paired caption ship in the example dataset. +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/diffsynth_example_dataset", + local_dir="data/diffsynth_example_dataset", + allow_file_pattern="lingbot_video/lingbot-video-moe-30b-a3b_ti2v/*", +) +base = "data/diffsynth_example_dataset/lingbot_video/lingbot-video-moe-30b-a3b_ti2v" +with open(os.path.join(base, "ti2v_example.json"), "r", encoding="utf-8") as f: + caption = json.load(f) +input_image = Image.open(os.path.join(base, "ti2v_first_frame.png")).convert("RGB") + +video = pipe( + prompt=caption, + negative_prompt=pipe.default_negative_prompt, + input_image=input_image, + height=480, width=832, num_frames=81, + num_inference_steps=40, cfg_scale=3.0, + seed=0, +) +save_video(video, "video_lingbot-video-moe-30b-a3b_ti2v.mp4", fps=15, quality=10) diff --git a/examples/ltx2/README.md b/examples/ltx2/README.md new file mode 100644 index 0000000000000000000000000000000000000000..042f668c83d226097c67c4a6919232a8c2c7ba58 --- /dev/null +++ b/examples/ltx2/README.md @@ -0,0 +1,3 @@ +English Document: https://diffsynth-studio-doc.readthedocs.io/en/latest/Model_Details/LTX-2.html + +中文文档：https://diffsynth-studio-doc.readthedocs.io/zh-cn/latest/Model_Details/LTX-2.html diff --git a/examples/ltx2/model_inference/JoyAI-Echo-T2AV.py b/examples/ltx2/model_inference/JoyAI-Echo-T2AV.py new file mode 100644 index 0000000000000000000000000000000000000000..cee163e5b30414746eb5ca85d0aba97fdcf3e321 --- /dev/null +++ b/examples/ltx2/model_inference/JoyAI-Echo-T2AV.py @@ -0,0 +1,47 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cuda", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} + +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="jd-opensource/JoyAI-Echo", origin_file_pattern="JoyAI-Echo-release.safetensors", **vram_config), + + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), +) + +prompt = "A girl is very happy, she is speaking: “I enjoy working with Diffsynth-Studio, it's a perfect framework.”" + +negative_prompt = "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +height, width, num_frames = 736, 1280, 241 +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + use_distilled_pipeline=True, + tiled=True, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='joyai_echo.mp4', + fps=24, + audio_sample_rate=pipe.audio_vocoder.output_sampling_rate, +) diff --git a/examples/ltx2/model_inference/LTX-2-I2AV-DistilledPipeline.py b/examples/ltx2/model_inference/LTX-2-I2AV-DistilledPipeline.py new file mode 100644 index 0000000000000000000000000000000000000000..39623dd28a3c309a6877fdc24c9b2378b65fa8d1 --- /dev/null +++ b/examples/ltx2/model_inference/LTX-2-I2AV-DistilledPipeline.py @@ -0,0 +1,74 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 +from PIL import Image +from modelscope import dataset_snapshot_download + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cuda", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="transformer_distilled.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="text_encoder_post_modules.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vocoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_encoder.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-spatial-upscaler-x2-1.0.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), +) + +prompt = "A girl is very happy, she is speaking: “I enjoy working with Diffsynth-Studio, it's a perfect framework.”" +negative_prompt = ( + "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, " + "grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, " + "deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, " + "wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of " + "field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent " + "lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny " + "valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, " + "mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, " + "off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward " + "pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, " + "inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +) +height, width, num_frames = 512 * 2, 768 * 2, 121 +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/examples_in_diffsynth", + local_dir="./", + allow_file_pattern=["data/examples/ltx-2/first_frame.jpg"] +) +image = Image.open("data/examples/ltx-2/first_frame.jpg").convert("RGB").resize((width, height)) +# first frame +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + tiled=True, + use_distilled_pipeline=True, + input_images=[image], + input_images_indexes=[0], + input_images_strength=1.0, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2_distilled_i2av_first.mp4', + fps=24, + audio_sample_rate=24000, +) diff --git a/examples/ltx2/model_inference/LTX-2-I2AV-OneStage.py b/examples/ltx2/model_inference/LTX-2-I2AV-OneStage.py new file mode 100644 index 0000000000000000000000000000000000000000..87fb682bc17ad2a6d0af4b70415f725fbb7c5b77 --- /dev/null +++ b/examples/ltx2/model_inference/LTX-2-I2AV-OneStage.py @@ -0,0 +1,60 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 +from PIL import Image +from modelscope import dataset_snapshot_download + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cuda", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="transformer.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="text_encoder_post_modules.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vocoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_encoder.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), +) +prompt = "A girl is very happy, she is speaking: “I enjoy working with Diffsynth-Studio, it's a perfect framework.”" +negative_prompt = "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +height, width, num_frames = 512, 768, 121 +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/examples_in_diffsynth", + local_dir="./", + allow_file_pattern=["data/examples/ltx-2/first_frame.jpg"] +) +image = Image.open("data/examples/ltx-2/first_frame.jpg").convert("RGB").resize((width, height)) +# first frame +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + tiled=False, + input_images=[image], + input_images_indexes=[0], + input_images_strength=1.0, + num_inference_steps=40, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2_onestage_i2av_first.mp4', + fps=24, + audio_sample_rate=24000, +) diff --git a/examples/ltx2/model_inference/LTX-2-I2AV-TwoStage.py b/examples/ltx2/model_inference/LTX-2-I2AV-TwoStage.py new file mode 100644 index 0000000000000000000000000000000000000000..0465803eb686be7f14fc91d48db91f2404622e0f --- /dev/null +++ b/examples/ltx2/model_inference/LTX-2-I2AV-TwoStage.py @@ -0,0 +1,76 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 +from PIL import Image +from modelscope import dataset_snapshot_download + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cuda", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="transformer.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="text_encoder_post_modules.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vocoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_encoder.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-spatial-upscaler-x2-1.0.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), + stage2_lora_config=ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-19b-distilled-lora-384.safetensors"), +) + +prompt = "A girl is very happy, she is speaking: “I enjoy working with Diffsynth-Studio, it's a perfect framework.”" +negative_prompt = ( + "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, " + "grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, " + "deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, " + "wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of " + "field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent " + "lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny " + "valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, " + "mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, " + "off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward " + "pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, " + "inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +) +height, width, num_frames = 512 * 2, 768 * 2, 121 +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/examples_in_diffsynth", + local_dir="./", + allow_file_pattern=["data/examples/ltx-2/first_frame.jpg"] +) +image = Image.open("data/examples/ltx-2/first_frame.jpg").convert("RGB").resize((width, height)) +# first frame +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=42, + height=height, + width=width, + num_frames=num_frames, + tiled=True, + use_two_stage_pipeline=True, + num_inference_steps=40, + input_images=[image], + input_images_indexes=[0], + input_images_strength=1.0, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2_twostage_i2av_first.mp4', + fps=24, + audio_sample_rate=24000, +) diff --git a/examples/ltx2/model_inference/LTX-2-T2AV-Camera-Control-Dolly-In.py b/examples/ltx2/model_inference/LTX-2-T2AV-Camera-Control-Dolly-In.py new file mode 100644 index 0000000000000000000000000000000000000000..99328048fb9026c54d38b53723fe123c50a0faf3 --- /dev/null +++ b/examples/ltx2/model_inference/LTX-2-T2AV-Camera-Control-Dolly-In.py @@ -0,0 +1,67 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cuda", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="transformer.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="text_encoder_post_modules.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vocoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_encoder.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-spatial-upscaler-x2-1.0.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), + stage2_lora_config=ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-19b-distilled-lora-384.safetensors"), +) +pipe.load_lora( + pipe.dit, + ModelConfig(model_id="Lightricks/LTX-2-19b-LoRA-Camera-Control-Dolly-In", origin_file_pattern="ltx-2-19b-lora-camera-control-dolly-in.safetensors"), +) + +prompt = "Dolly-in shot: A cheerful girl smiles brightly and says, 'I enjoy working with Diffsynth-Studio, it's a perfect framework.' The camera smoothly moves closer to her face, highlighting her enthusiasm and sincerity." +negative_prompt = ( + "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, " + "grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, " + "deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, " + "wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of " + "field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent " + "lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny " + "valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, " + "mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, " + "off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward " + "pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, " + "inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +) +height, width, num_frames = 512 * 2, 768 * 2, 121 +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + tiled=True, + use_two_stage_pipeline=True, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2_dolly_in_lora.mp4', + fps=24, + audio_sample_rate=24000, +) diff --git a/examples/ltx2/model_inference/LTX-2-T2AV-Camera-Control-Dolly-Left.py b/examples/ltx2/model_inference/LTX-2-T2AV-Camera-Control-Dolly-Left.py new file mode 100644 index 0000000000000000000000000000000000000000..4f0f273592c44227ac098233a9a7f572cffaf292 --- /dev/null +++ b/examples/ltx2/model_inference/LTX-2-T2AV-Camera-Control-Dolly-Left.py @@ -0,0 +1,67 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cuda", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="transformer.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="text_encoder_post_modules.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vocoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_encoder.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-spatial-upscaler-x2-1.0.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), + stage2_lora_config=ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-19b-distilled-lora-384.safetensors"), +) +pipe.load_lora( + pipe.dit, + ModelConfig(model_id="Lightricks/LTX-2-19b-LoRA-Camera-Control-Dolly-Left", origin_file_pattern="ltx-2-19b-lora-camera-control-dolly-left.safetensors"), +) + +prompt = "Dolly-left shot: A joyful young woman sits at a minimalist desk with a laptop running Diffsynth-Studio, code and generative visuals glowing on screen. She turns slightly toward the camera and says with a smile, 'I enjoy working with Diffsynth-Studio, it's a perfect framework.' As she speaks, the camera smoothly dollies left, revealing a wall of framed open-source project posters, a whiteboard covered in neural network sketches, and a shelf stacked with AI/graphics books beside her." +negative_prompt = ( + "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, " + "grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, " + "deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, " + "wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of " + "field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent " + "lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny " + "valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, " + "mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, " + "off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward " + "pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, " + "inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +) +height, width, num_frames = 512 * 2, 768 * 2, 121 +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + tiled=True, + use_two_stage_pipeline=True, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2_dolly_left_lora.mp4', + fps=24, + audio_sample_rate=24000, +) diff --git a/examples/ltx2/model_inference/LTX-2-T2AV-Camera-Control-Dolly-Out.py b/examples/ltx2/model_inference/LTX-2-T2AV-Camera-Control-Dolly-Out.py new file mode 100644 index 0000000000000000000000000000000000000000..3fdd2c36f4520380a819e3bef7866bac4abbfdbd --- /dev/null +++ b/examples/ltx2/model_inference/LTX-2-T2AV-Camera-Control-Dolly-Out.py @@ -0,0 +1,68 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cuda", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="transformer.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="text_encoder_post_modules.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vocoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_encoder.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-spatial-upscaler-x2-1.0.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), + stage2_lora_config=ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-19b-distilled-lora-384.safetensors"), +) +pipe.load_lora( + pipe.dit, + ModelConfig(model_id="Lightricks/LTX-2-19b-LoRA-Camera-Control-Dolly-Out", origin_file_pattern="ltx-2-19b-lora-camera-control-dolly-out.safetensors"), +) + +prompt = "Dolly-out shot: A joyful young woman smiles warmly and says: 'I enjoy working with Diffsynth-Studio, it's a perfect framework.' As she speaks, the camera slowly dollies out, revealing a bright, modern creative studio filled with plants, whiteboards full of diagrams, and soft natural light from large windows." + +negative_prompt = ( + "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, " + "grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, " + "deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, " + "wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of " + "field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent " + "lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny " + "valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, " + "mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, " + "off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward " + "pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, " + "inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +) +height, width, num_frames = 512 * 2, 768 * 2, 121 +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + tiled=True, + use_two_stage_pipeline=True, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path=f'ltx2_dolly_out.mp4', + fps=24, + audio_sample_rate=24000, +) diff --git a/examples/ltx2/model_inference/LTX-2-T2AV-Camera-Control-Dolly-Right.py b/examples/ltx2/model_inference/LTX-2-T2AV-Camera-Control-Dolly-Right.py new file mode 100644 index 0000000000000000000000000000000000000000..b4fd0996e7d59f36b265a2d3d2035b3c58a4c066 --- /dev/null +++ b/examples/ltx2/model_inference/LTX-2-T2AV-Camera-Control-Dolly-Right.py @@ -0,0 +1,67 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cuda", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="transformer.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="text_encoder_post_modules.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vocoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_encoder.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-spatial-upscaler-x2-1.0.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), + stage2_lora_config=ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-19b-distilled-lora-384.safetensors"), +) +pipe.load_lora( + pipe.dit, + ModelConfig(model_id="Lightricks/LTX-2-19b-LoRA-Camera-Control-Dolly-Right", origin_file_pattern="ltx-2-19b-lora-camera-control-dolly-right.safetensors"), +) + +prompt = "Dolly-right shot: A happy girl looks up and says happily, 'I enjoy working with Diffsynth-Studio, it's a perfect framework.' She sits before a sunlit café table, her open laptop displaying the Github interface. The camera glides right to show a barista crafting coffee in the background, shelves of artisan beans, and a chalkboard menu softly blurred in the bokeh." +negative_prompt = ( + "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, " + "grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, " + "deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, " + "wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of " + "field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent " + "lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny " + "valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, " + "mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, " + "off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward " + "pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, " + "inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +) +height, width, num_frames = 512 * 2, 768 * 2, 121 +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + tiled=True, + use_two_stage_pipeline=True, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2_dolly_right_lora.mp4', + fps=24, + audio_sample_rate=24000, +) diff --git a/examples/ltx2/model_inference/LTX-2-T2AV-Camera-Control-Jib-Down.py b/examples/ltx2/model_inference/LTX-2-T2AV-Camera-Control-Jib-Down.py new file mode 100644 index 0000000000000000000000000000000000000000..1eb2aced1e452311f7c7e28dd6a855aeed70e7eb --- /dev/null +++ b/examples/ltx2/model_inference/LTX-2-T2AV-Camera-Control-Jib-Down.py @@ -0,0 +1,77 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cuda", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="transformer.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="text_encoder_post_modules.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vocoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_encoder.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-spatial-upscaler-x2-1.0.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), + stage2_lora_config=ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-19b-distilled-lora-384.safetensors"), +) +pipe.load_lora( + pipe.dit, + ModelConfig(model_id="Lightricks/LTX-2-19b-LoRA-Camera-Control-Jib-Down", origin_file_pattern="ltx-2-19b-lora-camera-control-jib-down.safetensors"), +) +prompt = ( + "A girl is very happy, standing on a clean studio floor with soft ambient lighting. " + "She is speaking directly to the camera: “I enjoy working with Diffsynth-Studio, it's a perfect framework.” " + "The shot begins with a medium close-up framing her from the waist up. As the camera performs a smooth jib-down movement—" + "descending vertically downward—it gradually reveals more of the lower portion of the scene. " + "During the descent, the following elements become visible near the bottom of the frame: " + "- The polished concrete floor with subtle reflections of the girl’s shoes, " + "- A small branded mat labeled “Diffsynth-Studio” placed just beneath her feet, " + "- The lower part of a sleek workstation desk with a glowing logo on its front panel, partially hidden at the start but fully revealed as the camera lowers. " + "This downward motion provides a dynamic reveal of contextual details that reinforce the professional and creative environment, " + "while maintaining focus on the girl’s enthusiastic expression throughout." +) +negative_prompt = ( + "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, " + "grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, " + "deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, " + "wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of " + "field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent " + "lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny " + "valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, " + "mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, " + "off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward " + "pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, " + "inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +) +height, width, num_frames = 512 * 2, 768 * 2, 121 +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + tiled=True, + use_two_stage_pipeline=True, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2_camera_jib_down.mp4', + fps=24, + audio_sample_rate=24000, +) diff --git a/examples/ltx2/model_inference/LTX-2-T2AV-Camera-Control-Jib-Up.py b/examples/ltx2/model_inference/LTX-2-T2AV-Camera-Control-Jib-Up.py new file mode 100644 index 0000000000000000000000000000000000000000..cc6ac3936695facd3f605d7cfba6335dd6404cdf --- /dev/null +++ b/examples/ltx2/model_inference/LTX-2-T2AV-Camera-Control-Jib-Up.py @@ -0,0 +1,71 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cuda", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="transformer.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="text_encoder_post_modules.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vocoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_encoder.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-spatial-upscaler-x2-1.0.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), + stage2_lora_config=ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-19b-distilled-lora-384.safetensors"), +) +pipe.load_lora( + pipe.dit, + ModelConfig(model_id="Lightricks/LTX-2-19b-LoRA-Camera-Control-Jib-Up", origin_file_pattern="ltx-2-19b-lora-camera-control-jib-up.safetensors"), +) +prompt = ( + "A girl stands happily at a sleek desk with a glowing 'Diffsynth-Studio' logo, saying: “I enjoy working with Diffsynth-Studio, it's a perfect framework.” " + "The shot starts low—framing her waist, shoes, and a branded floor mat—and smoothly jib-ups upward. " + "As the camera rises, it reveals her smiling face, upper body, and behind her: a bright creative studio with wall art and a large window showing daylight sky. " + "The final frame fully shows the inspiring workspace above the initial view, ensuring spatial continuity." +) +negative_prompt = ( + "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, " + "grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, " + "deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, " + "wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of " + "field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent " + "lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny " + "valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, " + "mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, " + "off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward " + "pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, " + "inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +) +height, width, num_frames = 512 * 2, 768 * 2, 121 +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + tiled=True, + use_two_stage_pipeline=True, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2_camera_jib_up.mp4', + fps=24, + audio_sample_rate=24000, +) diff --git a/examples/ltx2/model_inference/LTX-2-T2AV-Camera-Control-Static.py b/examples/ltx2/model_inference/LTX-2-T2AV-Camera-Control-Static.py new file mode 100644 index 0000000000000000000000000000000000000000..2299bf7021b90bab5f4bcca59d8c42d79720396f --- /dev/null +++ b/examples/ltx2/model_inference/LTX-2-T2AV-Camera-Control-Static.py @@ -0,0 +1,66 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cuda", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="transformer.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="text_encoder_post_modules.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vocoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_encoder.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-spatial-upscaler-x2-1.0.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), + stage2_lora_config=ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-19b-distilled-lora-384.safetensors"), +) +pipe.load_lora( + pipe.dit, + ModelConfig(model_id="Lightricks/LTX-2-19b-LoRA-Camera-Control-Static", origin_file_pattern="ltx-2-19b-lora-camera-control-static.safetensors"), +) +prompt = "A beautiful sunset over the ocean." +negative_prompt = ( + "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, " + "grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, " + "deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, " + "wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of " + "field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent " + "lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny " + "valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, " + "mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, " + "off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward " + "pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, " + "inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +) +height, width, num_frames = 512 * 2, 768 * 2, 121 +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + tiled=True, + use_two_stage_pipeline=True, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2_camera_static.mp4', + fps=24, + audio_sample_rate=24000, +) diff --git a/examples/ltx2/model_inference/LTX-2-T2AV-DistilledPipeline.py b/examples/ltx2/model_inference/LTX-2-T2AV-DistilledPipeline.py new file mode 100644 index 0000000000000000000000000000000000000000..78b537b2282943f7b601b17c58d75ce3f92d8c0e --- /dev/null +++ b/examples/ltx2/model_inference/LTX-2-T2AV-DistilledPipeline.py @@ -0,0 +1,62 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cuda", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="transformer_distilled.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="text_encoder_post_modules.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vocoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_encoder.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-spatial-upscaler-x2-1.0.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), +) + +prompt = "A girl is very happy, she is speaking: “I enjoy working with Diffsynth-Studio, it's a perfect framework.”" +negative_prompt = ( + "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, " + "grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, " + "deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, " + "wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of " + "field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent " + "lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny " + "valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, " + "mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, " + "off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward " + "pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, " + "inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +) +height, width, num_frames = 512 * 2, 768 * 2, 121 +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + tiled=True, + use_distilled_pipeline=True, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2_distilled.mp4', + fps=24, + audio_sample_rate=24000, +) diff --git a/examples/ltx2/model_inference/LTX-2-T2AV-IC-LoRA-Detailer.py b/examples/ltx2/model_inference/LTX-2-T2AV-IC-LoRA-Detailer.py new file mode 100644 index 0000000000000000000000000000000000000000..687e2161f7c1c7958330be888b446cba2ba11510 --- /dev/null +++ b/examples/ltx2/model_inference/LTX-2-T2AV-IC-LoRA-Detailer.py @@ -0,0 +1,77 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 +from diffsynth.utils.data import VideoData +from modelscope import dataset_snapshot_download + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cuda", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="transformer.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="text_encoder_post_modules.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vocoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_encoder.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-spatial-upscaler-x2-1.0.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), + stage2_lora_config=ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-19b-distilled-lora-384.safetensors"), +) +pipe.load_lora(pipe.dit, ModelConfig(model_id="Lightricks/LTX-2-19b-IC-LoRA-Detailer", origin_file_pattern="ltx-2-19b-ic-lora-detailer.safetensors")) +dataset_snapshot_download("DiffSynth-Studio/example_video_dataset", allow_file_pattern="ltx2/*", local_dir="data/example_video_dataset") + +prompt = "[VISUAL]:Two cute orange cats, wearing boxing gloves, stand on a boxing ring and fight each other. [SOUNDS]:the sound of two cats boxing" +negative_prompt = ( + "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, " + "grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, " + "deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, " + "wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of " + "field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent " + "lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny " + "valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, " + "mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, " + "off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward " + "pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, " + "inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +) +height, width, num_frames = 512 * 2, 768 * 2, 121 +ref_scale_factor = 1 +frame_rate = 24 +# the frame rate of the video should better be the same with the reference video +# the spatial resolution of the first frame should be the resolution of stage 1 video generation divided by ref_scale_factor +input_video = VideoData("data/example_video_dataset/ltx2/video1.mp4", height=height // ref_scale_factor // 2, width=width // ref_scale_factor // 2) +input_video = input_video.raw_data() +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + frame_rate=frame_rate, + in_context_videos=[input_video], + in_context_downsample_factor=ref_scale_factor, + tiled=True, + use_two_stage_pipeline=True, + clear_lora_before_state_two=True, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2_twostage_iclora.mp4', + fps=frame_rate, + audio_sample_rate=24000, +) diff --git a/examples/ltx2/model_inference/LTX-2-T2AV-IC-LoRA-Union-Control.py b/examples/ltx2/model_inference/LTX-2-T2AV-IC-LoRA-Union-Control.py new file mode 100644 index 0000000000000000000000000000000000000000..30213061e82d2f92e1cf0102b1082dab08432828 --- /dev/null +++ b/examples/ltx2/model_inference/LTX-2-T2AV-IC-LoRA-Union-Control.py @@ -0,0 +1,77 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 +from diffsynth.utils.data import VideoData +from modelscope import dataset_snapshot_download + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cuda", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="transformer.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="text_encoder_post_modules.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vocoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_encoder.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-spatial-upscaler-x2-1.0.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), + stage2_lora_config=ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-19b-distilled-lora-384.safetensors"), +) +pipe.load_lora(pipe.dit, ModelConfig(model_id="Lightricks/LTX-2-19b-IC-LoRA-Union-Control", origin_file_pattern="ltx-2-19b-ic-lora-union-control-ref0.5.safetensors")) +dataset_snapshot_download("DiffSynth-Studio/example_video_dataset", allow_file_pattern="ltx2/*", local_dir="data/example_video_dataset") + +prompt = "[VISUAL]:Two cute orange cats, wearing boxing gloves, stand on a boxing ring and fight each other. [SOUNDS]:the sound of two cats boxing" +negative_prompt = ( + "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, " + "grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, " + "deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, " + "wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of " + "field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent " + "lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny " + "valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, " + "mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, " + "off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward " + "pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, " + "inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +) +height, width, num_frames = 512 * 2, 768 * 2, 121 +ref_scale_factor = 2 +frame_rate = 24 +# the frame rate of the video should better be the same with the reference video +# the spatial resolution of the first frame should be the resolution of stage 1 video generation divided by ref_scale_factor +input_video = VideoData("data/example_video_dataset/ltx2/depth_video.mp4", height=height // ref_scale_factor // 2, width=width // ref_scale_factor // 2) +input_video = input_video.raw_data() +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + frame_rate=frame_rate, + in_context_videos=[input_video], + in_context_downsample_factor=ref_scale_factor, + tiled=True, + use_two_stage_pipeline=True, + clear_lora_before_state_two=True, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2_twostage_iclora.mp4', + fps=frame_rate, + audio_sample_rate=24000, +) diff --git a/examples/ltx2/model_inference/LTX-2-T2AV-OneStage.py b/examples/ltx2/model_inference/LTX-2-T2AV-OneStage.py new file mode 100644 index 0000000000000000000000000000000000000000..9e562097532aeb88dab32042f1a9a6e509a56cdf --- /dev/null +++ b/examples/ltx2/model_inference/LTX-2-T2AV-OneStage.py @@ -0,0 +1,65 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cuda", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +""" +Offical model repo: https://www.modelscope.cn/models/Lightricks/LTX-2 +Repackaged model repo: https://www.modelscope.cn/models/DiffSynth-Studio/LTX-2-Repackage +For base models of LTX-2, offical checkpoint (with model config ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-19b-dev.safetensors")) +and repackaged checkpoints (with model config ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="*.safetensors")) are both supported. +We have repackeged the official checkpoints in DiffSynth-Studio/LTX-2-Repackage repo to support separate loading of different submodules, +and avoid redundant memory usage when users only want to use part of the model. +""" +# use the repackaged modelconfig from "DiffSynth-Studio/LTX-2-Repackage" to avoid redundant model loading +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="transformer.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="text_encoder_post_modules.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vocoder.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), +) +# use the following modelconfig if you want to initialize model from offical checkpoints from "Lightricks/LTX-2" +# pipe = LTX2AudioVideoPipeline.from_pretrained( +# torch_dtype=torch.bfloat16, +# device="cuda", +# model_configs=[ +# ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), +# ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-19b-dev.safetensors", **vram_config), +# ], +# tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), +# ) +prompt = "A girl is very happy, she is speaking: “I enjoy working with Diffsynth-Studio, it's a perfect framework.”" +negative_prompt = "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +height, width, num_frames = 512, 768, 121 +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + tiled=True, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2_onestage.mp4', + fps=24, + audio_sample_rate=24000, +) diff --git a/examples/ltx2/model_inference/LTX-2-T2AV-TwoStage.py b/examples/ltx2/model_inference/LTX-2-T2AV-TwoStage.py new file mode 100644 index 0000000000000000000000000000000000000000..e08ed503cd2f78988447506447233b8eb74e9768 --- /dev/null +++ b/examples/ltx2/model_inference/LTX-2-T2AV-TwoStage.py @@ -0,0 +1,83 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cuda", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +""" +Offical model repo: https://www.modelscope.cn/models/Lightricks/LTX-2 +Repackaged model repo: https://www.modelscope.cn/models/DiffSynth-Studio/LTX-2-Repackage +For base models of LTX-2, offical checkpoint (with model config ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-19b-dev.safetensors")) +and repackaged checkpoints (with model config ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="*.safetensors")) are both supported. +We have repackeged the official checkpoints in DiffSynth-Studio/LTX-2-Repackage repo to support separate loading of different submodules, +and avoid redundant memory usage when users only want to use part of the model. +""" +# use the repackaged modelconfig from "DiffSynth-Studio/LTX-2-Repackage" to avoid redundant model loading +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="transformer.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="text_encoder_post_modules.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vocoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_encoder.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-spatial-upscaler-x2-1.0.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), + stage2_lora_config=ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-19b-distilled-lora-384.safetensors"), +) +# use the following modelconfig if you want to initialize model from offical checkpoints from "Lightricks/LTX-2" +# pipe = LTX2AudioVideoPipeline.from_pretrained( +# torch_dtype=torch.bfloat16, +# device="cuda", +# model_configs=[ +# ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), +# ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-19b-dev.safetensors", **vram_config), +# ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-spatial-upscaler-x2-1.0.safetensors", **vram_config), +# ], +# tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), +# stage2_lora_config=ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-19b-distilled-lora-384.safetensors"), +# ) +prompt = "A girl is very happy, she is speaking: “I enjoy working with Diffsynth-Studio, it's a perfect framework.”" +negative_prompt = ( + "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, " + "grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, " + "deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, " + "wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of " + "field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent " + "lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny " + "valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, " + "mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, " + "off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward " + "pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, " + "inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +) +height, width, num_frames = 512 * 2, 768 * 2, 121 +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + tiled=True, + use_two_stage_pipeline=True, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2_twostage.mp4', + fps=24, + audio_sample_rate=24000, +) diff --git a/examples/ltx2/model_inference/LTX-2.3-A2V-TwoStage.py b/examples/ltx2/model_inference/LTX-2.3-A2V-TwoStage.py new file mode 100644 index 0000000000000000000000000000000000000000..ad0b91834cedd7e58d5a533a5f70eb7697e1a87c --- /dev/null +++ b/examples/ltx2/model_inference/LTX-2.3-A2V-TwoStage.py @@ -0,0 +1,66 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 +from diffsynth.utils.data.audio import read_audio +from modelscope import dataset_snapshot_download + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cuda", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2.3", origin_file_pattern="ltx-2.3-22b-dev.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2.3", origin_file_pattern="ltx-2.3-spatial-upscaler-x2-1.0.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), + stage2_lora_config=ModelConfig(model_id="Lightricks/LTX-2.3", origin_file_pattern="ltx-2.3-22b-distilled-lora-384.safetensors"), +) + +dataset_snapshot_download("DiffSynth-Studio/example_video_dataset", allow_file_pattern="ltx2/*", local_dir="data/example_video_dataset") +prompt = "A beautiful woman with a flower crown is singing happily under a blooming cherry tree." +negative_prompt = ( + "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, " + "grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, " + "deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, " + "wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of " + "field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent " + "lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny " + "valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, " + "mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, " + "off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward " + "pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, " + "inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +) +height, width, num_frames, frame_rate = 512 * 2, 768 * 2, 121, 24 +duration = num_frames / frame_rate +audio, audio_sample_rate = read_audio("data/example_video_dataset/ltx2/sing.MP3", start_time=1, duration=duration) +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + retake_audio=audio, + audio_sample_rate=audio_sample_rate, + seed=43, + height=height, + width=width, + num_frames=num_frames, + frame_rate=frame_rate, + tiled=True, + use_two_stage_pipeline=True, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2.3_twostage_a2v.mp4', + fps=frame_rate, + audio_sample_rate=pipe.audio_vocoder.output_sampling_rate, +) diff --git a/examples/ltx2/model_inference/LTX-2.3-I2AV-DistilledPipeline.py b/examples/ltx2/model_inference/LTX-2.3-I2AV-DistilledPipeline.py new file mode 100644 index 0000000000000000000000000000000000000000..d23460f5e7a39eab5dbca3f7a1f41ba6986f22b7 --- /dev/null +++ b/examples/ltx2/model_inference/LTX-2.3-I2AV-DistilledPipeline.py @@ -0,0 +1,70 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 +from PIL import Image +from modelscope import dataset_snapshot_download + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cuda", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2.3", origin_file_pattern="ltx-2.3-22b-distilled.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2.3", origin_file_pattern="ltx-2.3-spatial-upscaler-x2-1.0.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), +) + + +prompt = "A girl is very happy, she is speaking: “I enjoy working with Diffsynth-Studio, it's a perfect framework.”" +negative_prompt = ( + "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, " + "grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, " + "deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, " + "wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of " + "field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent " + "lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny " + "valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, " + "mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, " + "off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward " + "pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, " + "inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +) +height, width, num_frames = 512 * 2, 768 * 2, 121 +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/examples_in_diffsynth", + local_dir="./", + allow_file_pattern=["data/examples/ltx-2/first_frame.jpg"] +) +image = Image.open("data/examples/ltx-2/first_frame.jpg").convert("RGB").resize((width, height)) +# first frame +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + tiled=True, + use_distilled_pipeline=True, + input_images=[image], + input_images_indexes=[0], + input_images_strength=1.0, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2.3_distilled_i2av_first.mp4', + fps=24, + audio_sample_rate=pipe.audio_vocoder.output_sampling_rate, +) diff --git a/examples/ltx2/model_inference/LTX-2.3-I2AV-OneStage.py b/examples/ltx2/model_inference/LTX-2.3-I2AV-OneStage.py new file mode 100644 index 0000000000000000000000000000000000000000..7e76f0e19cb3bcbd1924f23fa7ae72f082264f86 --- /dev/null +++ b/examples/ltx2/model_inference/LTX-2.3-I2AV-OneStage.py @@ -0,0 +1,55 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 +from PIL import Image +from modelscope import dataset_snapshot_download + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cuda", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2.3", origin_file_pattern="ltx-2.3-22b-dev.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), +) + +prompt = "A girl is very happy, she is speaking: “I enjoy working with Diffsynth-Studio, it's a perfect framework.”" +negative_prompt = "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +height, width, num_frames = 512, 768, 121 +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/examples_in_diffsynth", + local_dir="./", + allow_file_pattern=["data/examples/ltx-2/first_frame.jpg"] +) +image = Image.open("data/examples/ltx-2/first_frame.jpg").convert("RGB").resize((width, height)) +# first frame +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + tiled=False, + input_images=[image], + input_images_indexes=[0], + input_images_strength=1.0, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2.3_onestage_i2av_first.mp4', + fps=24, + audio_sample_rate=pipe.audio_vocoder.output_sampling_rate, +) diff --git a/examples/ltx2/model_inference/LTX-2.3-I2AV-TwoStage.py b/examples/ltx2/model_inference/LTX-2.3-I2AV-TwoStage.py new file mode 100644 index 0000000000000000000000000000000000000000..0dd0854c5f4fff9e946c8a499f31f34841f6ec46 --- /dev/null +++ b/examples/ltx2/model_inference/LTX-2.3-I2AV-TwoStage.py @@ -0,0 +1,88 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 +from PIL import Image +from modelscope import dataset_snapshot_download + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cuda", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2.3", origin_file_pattern="ltx-2.3-22b-dev.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2.3", origin_file_pattern="ltx-2.3-spatial-upscaler-x2-1.0.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), + stage2_lora_config=ModelConfig(model_id="Lightricks/LTX-2.3", origin_file_pattern="ltx-2.3-22b-distilled-lora-384.safetensors"), +) + +prompt = "Two cute orange cats, wearing boxing gloves, stand in a boxing ring and fight each other. They are punching each other fast and yelling: 'I will win!'" +negative_prompt = ( + "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, " + "grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, " + "deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, " + "wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of " + "field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent " + "lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny " + "valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, " + "mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, " + "off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward " + "pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, " + "inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +) +height, width, num_frames = 512 * 2, 768 * 2, 121 +dataset_snapshot_download("DiffSynth-Studio/example_video_dataset", allow_file_pattern="ltx2/*", local_dir="data/example_video_dataset") +first_frame = Image.open("data/example_video_dataset/ltx2/first_frame.png").convert("RGB").resize((width, height)) +last_frame = Image.open("data/example_video_dataset/ltx2/last_frame.png").convert("RGB").resize((width, height)) +# first frame +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=42, + height=height, + width=width, + num_frames=num_frames, + tiled=True, + use_two_stage_pipeline=True, + input_images=[first_frame], + input_images_indexes=[0], + input_images_strength=1.0, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2.3_twostage_i2av_first.mp4', + fps=24, +) +pipe.clear_lora() + +# This example uses the first and last frames for demonstration. However, you can use any frames by setting input_images and input_indexes. Note that input_indexes must be within the range of num_frames. +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=42, + height=height, + width=width, + num_frames=num_frames, + tiled=True, + use_two_stage_pipeline=True, + input_images=[first_frame, last_frame], + input_images_indexes=[0, num_frames-1], + input_images_strength=1.0, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2.3_twostage_i2av_first_last.mp4', + fps=24, +) diff --git a/examples/ltx2/model_inference/LTX-2.3-T2AV-DistilledPipeline.py b/examples/ltx2/model_inference/LTX-2.3-T2AV-DistilledPipeline.py new file mode 100644 index 0000000000000000000000000000000000000000..047a0b0d511f060e02c8449a14e1deb07a764f22 --- /dev/null +++ b/examples/ltx2/model_inference/LTX-2.3-T2AV-DistilledPipeline.py @@ -0,0 +1,57 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cuda", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2.3", origin_file_pattern="ltx-2.3-22b-distilled.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2.3", origin_file_pattern="ltx-2.3-spatial-upscaler-x2-1.0.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), +) + +prompt = "A girl is very happy, she is speaking: “I enjoy working with Diffsynth-Studio, it's a perfect framework.”" +negative_prompt = ( + "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, " + "grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, " + "deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, " + "wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of " + "field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent " + "lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny " + "valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, " + "mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, " + "off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward " + "pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, " + "inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +) +height, width, num_frames = 512 * 2, 768 * 2, 121 +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + tiled=True, + use_distilled_pipeline=True, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2.3_distilled.mp4', + fps=24, + audio_sample_rate=pipe.audio_vocoder.output_sampling_rate, +) diff --git a/examples/ltx2/model_inference/LTX-2.3-T2AV-IC-LoRA-Motion-Track-Control.py b/examples/ltx2/model_inference/LTX-2.3-T2AV-IC-LoRA-Motion-Track-Control.py new file mode 100644 index 0000000000000000000000000000000000000000..5708f0d6fee46cb82c742521c6849dea10b91edd --- /dev/null +++ b/examples/ltx2/model_inference/LTX-2.3-T2AV-IC-LoRA-Motion-Track-Control.py @@ -0,0 +1,71 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 +from modelscope import dataset_snapshot_download +from diffsynth.utils.data import VideoData + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cuda", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2.3", origin_file_pattern="ltx-2.3-22b-dev.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2.3", origin_file_pattern="ltx-2.3-spatial-upscaler-x2-1.0.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), + stage2_lora_config=ModelConfig(model_id="Lightricks/LTX-2.3", origin_file_pattern="ltx-2.3-22b-distilled-lora-384.safetensors"), +) +pipe.load_lora(pipe.dit, ModelConfig(model_id="Lightricks/LTX-2.3-22b-IC-LoRA-Motion-Track-Control", origin_file_pattern="ltx-2.3-22b-ic-lora-motion-track-control-ref0.5.safetensors")) +dataset_snapshot_download("DiffSynth-Studio/example_video_dataset", allow_file_pattern="ltx2/*", local_dir="data/example_video_dataset") +prompt = "[VISUAL]:Two cute orange cats, wearing boxing gloves, stand on a boxing ring and fight each other. [SOUNDS]:the sound of two cats boxing" +negative_prompt = ( + "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, " + "grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, " + "deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, " + "wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of " + "field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent " + "lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny " + "valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, " + "mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, " + "off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward " + "pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, " + "inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +) +height, width, num_frames = 512 * 2, 768 * 2, 121 +ref_scale_factor = 2 +frame_rate = 24 +input_image = VideoData("data/example_video_dataset/ltx2/video1.mp4", height=height // ref_scale_factor // 2, width=width // ref_scale_factor // 2)[0] +input_video = VideoData("data/example_video_dataset/ltx2/spatial_tracker_v2.mp4", height=height // ref_scale_factor // 2, width=width // ref_scale_factor // 2).raw_data() +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + frame_rate=frame_rate, + in_context_videos=[input_video], + in_context_downsample_factor=ref_scale_factor, + input_images=[input_image], + input_images_indexes=[0], + tiled=True, + use_two_stage_pipeline=True, + clear_lora_before_state_two=True, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2.3_ic_lora.mp4', + fps=frame_rate, + audio_sample_rate=pipe.audio_vocoder.output_sampling_rate, +) diff --git a/examples/ltx2/model_inference/LTX-2.3-T2AV-IC-LoRA-Union-Control.py b/examples/ltx2/model_inference/LTX-2.3-T2AV-IC-LoRA-Union-Control.py new file mode 100644 index 0000000000000000000000000000000000000000..25883f6814381872e3e70eea61d26f91e70aceaa --- /dev/null +++ b/examples/ltx2/model_inference/LTX-2.3-T2AV-IC-LoRA-Union-Control.py @@ -0,0 +1,68 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 +from modelscope import dataset_snapshot_download +from diffsynth.utils.data import VideoData + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cuda", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2.3", origin_file_pattern="ltx-2.3-22b-dev.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2.3", origin_file_pattern="ltx-2.3-spatial-upscaler-x2-1.0.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), + stage2_lora_config=ModelConfig(model_id="Lightricks/LTX-2.3", origin_file_pattern="ltx-2.3-22b-distilled-lora-384.safetensors"), +) +pipe.load_lora(pipe.dit, ModelConfig(model_id="Lightricks/LTX-2.3-22b-IC-LoRA-Union-Control", origin_file_pattern="ltx-2.3-22b-ic-lora-union-control-ref0.5.safetensors")) +dataset_snapshot_download("DiffSynth-Studio/example_video_dataset", allow_file_pattern="ltx2/*", local_dir="data/example_video_dataset") +prompt = "[VISUAL]:Two cute orange cats, wearing boxing gloves, stand on a boxing ring and fight each other. [SOUNDS]:the sound of two cats boxing" +negative_prompt = ( + "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, " + "grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, " + "deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, " + "wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of " + "field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent " + "lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny " + "valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, " + "mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, " + "off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward " + "pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, " + "inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +) +height, width, num_frames = 512 * 2, 768 * 2, 121 +ref_scale_factor = 2 +frame_rate = 24 +input_video = VideoData("data/example_video_dataset/ltx2/depth_video.mp4", height=height // ref_scale_factor // 2, width=width // ref_scale_factor // 2).raw_data() +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + frame_rate=frame_rate, + in_context_videos=[input_video], + in_context_downsample_factor=ref_scale_factor, + tiled=True, + use_two_stage_pipeline=True, + clear_lora_before_state_two=True, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2.3_ic_lora.mp4', + fps=frame_rate, + audio_sample_rate=pipe.audio_vocoder.output_sampling_rate, +) diff --git a/examples/ltx2/model_inference/LTX-2.3-T2AV-OneStage.py b/examples/ltx2/model_inference/LTX-2.3-T2AV-OneStage.py new file mode 100644 index 0000000000000000000000000000000000000000..431138782d31b5dbbb62afc342c76ac1e0252cb6 --- /dev/null +++ b/examples/ltx2/model_inference/LTX-2.3-T2AV-OneStage.py @@ -0,0 +1,57 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cuda", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +# use the repackaged modelconfig from "DiffSynth-Studio/LTX-2.3-Repackage" to avoid redundant model loading +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2.3-Repackage", origin_file_pattern="transformer.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2.3-Repackage", origin_file_pattern="text_encoder_post_modules.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2.3-Repackage", origin_file_pattern="video_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2.3-Repackage", origin_file_pattern="audio_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2.3-Repackage", origin_file_pattern="audio_vocoder.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), +) +# use the following modelconfig if you want to initialize model from offical checkpoints from "Lightricks/LTX-2.3" +# pipe = LTX2AudioVideoPipeline.from_pretrained( +# torch_dtype=torch.bfloat16, +# device="cuda", +# model_configs=[ +# ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), +# ModelConfig(model_id="Lightricks/LTX-2.3", origin_file_pattern="ltx-2.3-22b-dev.safetensors", **vram_config), +# ], +# tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), +# ) +prompt = "A girl is very happy, she is speaking: “I enjoy working with Diffsynth-Studio, it's a perfect framework.”" +negative_prompt = "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +height, width, num_frames = 512, 768, 121 +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + tiled=True, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2.3_onestage.mp4', + fps=24, + audio_sample_rate=pipe.audio_vocoder.output_sampling_rate, +) diff --git a/examples/ltx2/model_inference/LTX-2.3-T2AV-TwoStage-Retake.py b/examples/ltx2/model_inference/LTX-2.3-T2AV-TwoStage-Retake.py new file mode 100644 index 0000000000000000000000000000000000000000..d241f69da9b67651e865c024e74abbdc3b5bf67f --- /dev/null +++ b/examples/ltx2/model_inference/LTX-2.3-T2AV-TwoStage-Retake.py @@ -0,0 +1,77 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 +from diffsynth.utils.data.audio import read_audio +from modelscope import dataset_snapshot_download +from diffsynth.utils.data import VideoData + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cuda", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2.3", origin_file_pattern="ltx-2.3-22b-dev.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2.3", origin_file_pattern="ltx-2.3-spatial-upscaler-x2-1.0.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), + stage2_lora_config=ModelConfig(model_id="Lightricks/LTX-2.3", origin_file_pattern="ltx-2.3-22b-distilled-lora-384.safetensors"), +) + +dataset_snapshot_download("DiffSynth-Studio/example_video_dataset", allow_file_pattern="ltx2/*", local_dir="data/example_video_dataset") +prompt = "A beautiful woman with a flower crown is singing happily under a blooming cherry tree. She sings: 'Mummy don't know daddy's getting hot. At the body shop'" +negative_prompt = ( + "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, " + "grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, " + "deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, " + "wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of " + "field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent " + "lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny " + "valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, " + "mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, " + "off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward " + "pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, " + "inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +) + +height, width, num_frames, frame_rate = 512 * 2, 768 * 2, 121, 24 +path = "data/example_video_dataset/ltx2/video2.mp4" +video = VideoData(path, height=height, width=width).raw_data()[:num_frames] +assert len(video) == num_frames, f"Input video has {len(video)} frames, but expected {num_frames} frames based on the specified num_frames argument." +duration = num_frames / frame_rate +audio, audio_sample_rate = read_audio(path) + +# Regenerate the video within time regions. You can specify different time regions for video frames and audio retake. +# retake regions are in seconds, and the example below retakes video frames in the time regions of [1s, 2s] and [3s, 4s], and retakes audio in the time regions of [0s, 1s] and [4s, 5s]. +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + retake_video=video, + retake_video_regions=[(1, 2), (3, 4)], + retake_audio=audio, + audio_sample_rate=audio_sample_rate, + retake_audio_regions=[(0, 1), (4, 5)], + seed=43, + height=height, + width=width, + num_frames=num_frames, + frame_rate=frame_rate, + tiled=True, + use_two_stage_pipeline=True, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2.3_twostage_retake.mp4', + fps=frame_rate, + audio_sample_rate=pipe.audio_vocoder.output_sampling_rate, +) diff --git a/examples/ltx2/model_inference/LTX-2.3-T2AV-TwoStage.py b/examples/ltx2/model_inference/LTX-2.3-T2AV-TwoStage.py new file mode 100644 index 0000000000000000000000000000000000000000..52345894e29ed3d1852a2c680521947da09c6613 --- /dev/null +++ b/examples/ltx2/model_inference/LTX-2.3-T2AV-TwoStage.py @@ -0,0 +1,57 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cuda", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2.3", origin_file_pattern="ltx-2.3-22b-dev.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2.3", origin_file_pattern="ltx-2.3-spatial-upscaler-x2-1.0.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), + stage2_lora_config=ModelConfig(model_id="Lightricks/LTX-2.3", origin_file_pattern="ltx-2.3-22b-distilled-lora-384.safetensors"), +) +prompt = "A girl is very happy, she is speaking: “I enjoy working with Diffsynth-Studio, it's a perfect framework.”" +negative_prompt = ( + "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, " + "grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, " + "deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, " + "wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of " + "field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent " + "lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny " + "valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, " + "mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, " + "off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward " + "pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, " + "inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +) +height, width, num_frames = 512 * 2, 768 * 2, 121 +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + tiled=True, + use_two_stage_pipeline=True, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2.3_twostage.mp4', + fps=24, + audio_sample_rate=pipe.audio_vocoder.output_sampling_rate, +) diff --git a/examples/ltx2/model_inference_low_vram/JoyAI-Echo-T2AV.py b/examples/ltx2/model_inference_low_vram/JoyAI-Echo-T2AV.py new file mode 100644 index 0000000000000000000000000000000000000000..4c8a30817fd3ffeb7a7c51415497fb9a2839ad0e --- /dev/null +++ b/examples/ltx2/model_inference_low_vram/JoyAI-Echo-T2AV.py @@ -0,0 +1,48 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 + +vram_config = { + "offload_dtype": torch.float8_e5m2, + "offload_device": "cpu", + "onload_dtype": torch.float8_e5m2, + "onload_device": "cpu", + "preparing_dtype": torch.float8_e5m2, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} + +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="jd-opensource/JoyAI-Echo", origin_file_pattern="JoyAI-Echo-release.safetensors", **vram_config), + + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), + vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 0.5, +) + +prompt = "A girl is very happy, she is speaking: “I enjoy working with Diffsynth-Studio, it's a perfect framework.”" + +negative_prompt = "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +height, width, num_frames = 736, 1280, 241 +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + use_distilled_pipeline=True, + tiled=True, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='joyai_echo.mp4', + fps=24, + audio_sample_rate=pipe.audio_vocoder.output_sampling_rate, +) diff --git a/examples/ltx2/model_inference_low_vram/LTX-2-I2AV-DistilledPipeline.py b/examples/ltx2/model_inference_low_vram/LTX-2-I2AV-DistilledPipeline.py new file mode 100644 index 0000000000000000000000000000000000000000..370e3710568aabc3e1ed3bd0ec928d1b7a64a295 --- /dev/null +++ b/examples/ltx2/model_inference_low_vram/LTX-2-I2AV-DistilledPipeline.py @@ -0,0 +1,75 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 +from PIL import Image +from modelscope import dataset_snapshot_download + +vram_config = { + "offload_dtype": torch.float8_e5m2, + "offload_device": "cpu", + "onload_dtype": torch.float8_e5m2, + "onload_device": "cpu", + "preparing_dtype": torch.float8_e5m2, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="transformer_distilled.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="text_encoder_post_modules.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vocoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_encoder.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-spatial-upscaler-x2-1.0.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), + vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 0.5, +) + +prompt = "A girl is very happy, she is speaking: “I enjoy working with Diffsynth-Studio, it's a perfect framework.”" +negative_prompt = ( + "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, " + "grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, " + "deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, " + "wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of " + "field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent " + "lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny " + "valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, " + "mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, " + "off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward " + "pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, " + "inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +) +height, width, num_frames = 512 * 2, 768 * 2, 121 +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/examples_in_diffsynth", + local_dir="./", + allow_file_pattern=["data/examples/ltx-2/first_frame.jpg"] +) +image = Image.open("data/examples/ltx-2/first_frame.jpg").convert("RGB").resize((width, height)) +# first frame +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + tiled=True, + use_distilled_pipeline=True, + input_images=[image], + input_images_indexes=[0], + input_images_strength=1.0, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2_distilled_i2av_first.mp4', + fps=24, + audio_sample_rate=24000, +) diff --git a/examples/ltx2/model_inference_low_vram/LTX-2-I2AV-OneStage.py b/examples/ltx2/model_inference_low_vram/LTX-2-I2AV-OneStage.py new file mode 100644 index 0000000000000000000000000000000000000000..3ff9932294a6aa68d3c77f383e568e7e1e4fc408 --- /dev/null +++ b/examples/ltx2/model_inference_low_vram/LTX-2-I2AV-OneStage.py @@ -0,0 +1,61 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 +from PIL import Image +from modelscope import dataset_snapshot_download + +vram_config = { + "offload_dtype": torch.float8_e5m2, + "offload_device": "cpu", + "onload_dtype": torch.float8_e5m2, + "onload_device": "cpu", + "preparing_dtype": torch.float8_e5m2, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="transformer.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="text_encoder_post_modules.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vocoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_encoder.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), + vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 0.5, +) +prompt = "A girl is very happy, she is speaking: “I enjoy working with Diffsynth-Studio, it's a perfect framework.”" +negative_prompt = "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +height, width, num_frames = 512, 768, 121 +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/examples_in_diffsynth", + local_dir="./", + allow_file_pattern=["data/examples/ltx-2/first_frame.jpg"] +) +image = Image.open("data/examples/ltx-2/first_frame.jpg").convert("RGB").resize((width, height)) +# first frame +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + tiled=False, + input_images=[image], + input_images_indexes=[0], + input_images_strength=1.0, + num_inference_steps=40, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2_onestage_i2av_first.mp4', + fps=24, + audio_sample_rate=24000, +) diff --git a/examples/ltx2/model_inference_low_vram/LTX-2-I2AV-TwoStage.py b/examples/ltx2/model_inference_low_vram/LTX-2-I2AV-TwoStage.py new file mode 100644 index 0000000000000000000000000000000000000000..912fa9309dab89b3d08b07a4d62211a58882987c --- /dev/null +++ b/examples/ltx2/model_inference_low_vram/LTX-2-I2AV-TwoStage.py @@ -0,0 +1,77 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 +from PIL import Image +from modelscope import dataset_snapshot_download + +vram_config = { + "offload_dtype": torch.float8_e5m2, + "offload_device": "cpu", + "onload_dtype": torch.float8_e5m2, + "onload_device": "cpu", + "preparing_dtype": torch.float8_e5m2, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="transformer.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="text_encoder_post_modules.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vocoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_encoder.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-spatial-upscaler-x2-1.0.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), + stage2_lora_config=ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-19b-distilled-lora-384.safetensors"), + vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 0.5, +) + +prompt = "A girl is very happy, she is speaking: “I enjoy working with Diffsynth-Studio, it's a perfect framework.”" +negative_prompt = ( + "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, " + "grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, " + "deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, " + "wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of " + "field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent " + "lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny " + "valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, " + "mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, " + "off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward " + "pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, " + "inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +) +height, width, num_frames = 512 * 2, 768 * 2, 121 +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/examples_in_diffsynth", + local_dir="./", + allow_file_pattern=["data/examples/ltx-2/first_frame.jpg"] +) +image = Image.open("data/examples/ltx-2/first_frame.jpg").convert("RGB").resize((width, height)) +# first frame +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=42, + height=height, + width=width, + num_frames=num_frames, + tiled=True, + use_two_stage_pipeline=True, + num_inference_steps=40, + input_images=[image], + input_images_indexes=[0], + input_images_strength=1.0, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2_twostage_i2av_first.mp4', + fps=24, + audio_sample_rate=24000, +) diff --git a/examples/ltx2/model_inference_low_vram/LTX-2-T2AV-Camera-Control-Dolly-In.py b/examples/ltx2/model_inference_low_vram/LTX-2-T2AV-Camera-Control-Dolly-In.py new file mode 100644 index 0000000000000000000000000000000000000000..0454dfd26fe610a0f5e24ee5eb98f276d14bbfb4 --- /dev/null +++ b/examples/ltx2/model_inference_low_vram/LTX-2-T2AV-Camera-Control-Dolly-In.py @@ -0,0 +1,67 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 + +vram_config = { + "offload_dtype": torch.float8_e5m2, + "offload_device": "cpu", + "onload_dtype": torch.float8_e5m2, + "onload_device": "cpu", + "preparing_dtype": torch.float8_e5m2, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="transformer.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="text_encoder_post_modules.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vocoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_encoder.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-spatial-upscaler-x2-1.0.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), + stage2_lora_config=ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-19b-distilled-lora-384.safetensors"), +) +pipe.load_lora( + pipe.dit, + ModelConfig(model_id="Lightricks/LTX-2-19b-LoRA-Camera-Control-Dolly-In", origin_file_pattern="ltx-2-19b-lora-camera-control-dolly-in.safetensors"), +) + +prompt = "Dolly-in shot: A cheerful girl smiles brightly and says, 'I enjoy working with Diffsynth-Studio, it's a perfect framework.' The camera smoothly moves closer to her face, highlighting her enthusiasm and sincerity." +negative_prompt = ( + "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, " + "grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, " + "deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, " + "wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of " + "field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent " + "lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny " + "valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, " + "mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, " + "off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward " + "pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, " + "inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +) +height, width, num_frames = 512 * 2, 768 * 2, 121 +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + tiled=True, + use_two_stage_pipeline=True, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2_dolly_in_lora.mp4', + fps=24, + audio_sample_rate=24000, +) diff --git a/examples/ltx2/model_inference_low_vram/LTX-2-T2AV-Camera-Control-Dolly-Left.py b/examples/ltx2/model_inference_low_vram/LTX-2-T2AV-Camera-Control-Dolly-Left.py new file mode 100644 index 0000000000000000000000000000000000000000..e66a2030989650c2fce52cb1e9cd819b4c241320 --- /dev/null +++ b/examples/ltx2/model_inference_low_vram/LTX-2-T2AV-Camera-Control-Dolly-Left.py @@ -0,0 +1,67 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 + +vram_config = { + "offload_dtype": torch.float8_e5m2, + "offload_device": "cpu", + "onload_dtype": torch.float8_e5m2, + "onload_device": "cpu", + "preparing_dtype": torch.float8_e5m2, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="transformer.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="text_encoder_post_modules.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vocoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_encoder.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-spatial-upscaler-x2-1.0.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), + stage2_lora_config=ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-19b-distilled-lora-384.safetensors"), +) +pipe.load_lora( + pipe.dit, + ModelConfig(model_id="Lightricks/LTX-2-19b-LoRA-Camera-Control-Dolly-Left", origin_file_pattern="ltx-2-19b-lora-camera-control-dolly-left.safetensors"), +) + +prompt = "Dolly-left shot: A joyful young woman sits at a minimalist desk with a laptop running Diffsynth-Studio, code and generative visuals glowing on screen. She turns slightly toward the camera and says with a smile, 'I enjoy working with Diffsynth-Studio, it's a perfect framework.' As she speaks, the camera smoothly dollies left, revealing a wall of framed open-source project posters, a whiteboard covered in neural network sketches, and a shelf stacked with AI/graphics books beside her." +negative_prompt = ( + "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, " + "grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, " + "deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, " + "wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of " + "field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent " + "lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny " + "valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, " + "mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, " + "off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward " + "pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, " + "inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +) +height, width, num_frames = 512 * 2, 768 * 2, 121 +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + tiled=True, + use_two_stage_pipeline=True, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2_dolly_left_lora.mp4', + fps=24, + audio_sample_rate=24000, +) diff --git a/examples/ltx2/model_inference_low_vram/LTX-2-T2AV-Camera-Control-Dolly-Out.py b/examples/ltx2/model_inference_low_vram/LTX-2-T2AV-Camera-Control-Dolly-Out.py new file mode 100644 index 0000000000000000000000000000000000000000..94af241236dc20e47dcc4f546cb5cc1916d9742b --- /dev/null +++ b/examples/ltx2/model_inference_low_vram/LTX-2-T2AV-Camera-Control-Dolly-Out.py @@ -0,0 +1,68 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 + +vram_config = { + "offload_dtype": torch.float8_e5m2, + "offload_device": "cpu", + "onload_dtype": torch.float8_e5m2, + "onload_device": "cpu", + "preparing_dtype": torch.float8_e5m2, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="transformer.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="text_encoder_post_modules.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vocoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_encoder.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-spatial-upscaler-x2-1.0.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), + stage2_lora_config=ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-19b-distilled-lora-384.safetensors"), +) +pipe.load_lora( + pipe.dit, + ModelConfig(model_id="Lightricks/LTX-2-19b-LoRA-Camera-Control-Dolly-Out", origin_file_pattern="ltx-2-19b-lora-camera-control-dolly-out.safetensors"), +) + +prompt = "Dolly-out shot: A joyful young woman smiles warmly and says: 'I enjoy working with Diffsynth-Studio, it's a perfect framework.' As she speaks, the camera slowly dollies out, revealing a bright, modern creative studio filled with plants, whiteboards full of diagrams, and soft natural light from large windows." + +negative_prompt = ( + "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, " + "grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, " + "deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, " + "wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of " + "field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent " + "lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny " + "valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, " + "mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, " + "off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward " + "pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, " + "inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +) +height, width, num_frames = 512 * 2, 768 * 2, 121 +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + tiled=True, + use_two_stage_pipeline=True, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path=f'ltx2_dolly_out.mp4', + fps=24, + audio_sample_rate=24000, +) diff --git a/examples/ltx2/model_inference_low_vram/LTX-2-T2AV-Camera-Control-Dolly-Right.py b/examples/ltx2/model_inference_low_vram/LTX-2-T2AV-Camera-Control-Dolly-Right.py new file mode 100644 index 0000000000000000000000000000000000000000..b7131706b1f79fce9f57ce3d97cccbda82447506 --- /dev/null +++ b/examples/ltx2/model_inference_low_vram/LTX-2-T2AV-Camera-Control-Dolly-Right.py @@ -0,0 +1,67 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 + +vram_config = { + "offload_dtype": torch.float8_e5m2, + "offload_device": "cpu", + "onload_dtype": torch.float8_e5m2, + "onload_device": "cpu", + "preparing_dtype": torch.float8_e5m2, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="transformer.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="text_encoder_post_modules.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vocoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_encoder.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-spatial-upscaler-x2-1.0.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), + stage2_lora_config=ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-19b-distilled-lora-384.safetensors"), +) +pipe.load_lora( + pipe.dit, + ModelConfig(model_id="Lightricks/LTX-2-19b-LoRA-Camera-Control-Dolly-Right", origin_file_pattern="ltx-2-19b-lora-camera-control-dolly-right.safetensors"), +) + +prompt = "Dolly-right shot: A happy girl looks up and says happily, 'I enjoy working with Diffsynth-Studio, it's a perfect framework.' She sits before a sunlit café table, her open laptop displaying the Github interface. The camera glides right to show a barista crafting coffee in the background, shelves of artisan beans, and a chalkboard menu softly blurred in the bokeh." +negative_prompt = ( + "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, " + "grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, " + "deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, " + "wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of " + "field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent " + "lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny " + "valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, " + "mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, " + "off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward " + "pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, " + "inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +) +height, width, num_frames = 512 * 2, 768 * 2, 121 +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + tiled=True, + use_two_stage_pipeline=True, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2_dolly_right_lora.mp4', + fps=24, + audio_sample_rate=24000, +) diff --git a/examples/ltx2/model_inference_low_vram/LTX-2-T2AV-Camera-Control-Jib-Down.py b/examples/ltx2/model_inference_low_vram/LTX-2-T2AV-Camera-Control-Jib-Down.py new file mode 100644 index 0000000000000000000000000000000000000000..fe1423cf60f645d1512cac4029106577ae9b01c9 --- /dev/null +++ b/examples/ltx2/model_inference_low_vram/LTX-2-T2AV-Camera-Control-Jib-Down.py @@ -0,0 +1,77 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 + +vram_config = { + "offload_dtype": torch.float8_e5m2, + "offload_device": "cpu", + "onload_dtype": torch.float8_e5m2, + "onload_device": "cpu", + "preparing_dtype": torch.float8_e5m2, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="transformer.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="text_encoder_post_modules.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vocoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_encoder.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-spatial-upscaler-x2-1.0.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), + stage2_lora_config=ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-19b-distilled-lora-384.safetensors"), +) +pipe.load_lora( + pipe.dit, + ModelConfig(model_id="Lightricks/LTX-2-19b-LoRA-Camera-Control-Jib-Down", origin_file_pattern="ltx-2-19b-lora-camera-control-jib-down.safetensors"), +) +prompt = ( + "A girl is very happy, standing on a clean studio floor with soft ambient lighting. " + "She is speaking directly to the camera: “I enjoy working with Diffsynth-Studio, it's a perfect framework.” " + "The shot begins with a medium close-up framing her from the waist up. As the camera performs a smooth jib-down movement—" + "descending vertically downward—it gradually reveals more of the lower portion of the scene. " + "During the descent, the following elements become visible near the bottom of the frame: " + "- The polished concrete floor with subtle reflections of the girl’s shoes, " + "- A small branded mat labeled “Diffsynth-Studio” placed just beneath her feet, " + "- The lower part of a sleek workstation desk with a glowing logo on its front panel, partially hidden at the start but fully revealed as the camera lowers. " + "This downward motion provides a dynamic reveal of contextual details that reinforce the professional and creative environment, " + "while maintaining focus on the girl’s enthusiastic expression throughout." +) +negative_prompt = ( + "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, " + "grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, " + "deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, " + "wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of " + "field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent " + "lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny " + "valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, " + "mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, " + "off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward " + "pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, " + "inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +) +height, width, num_frames = 512 * 2, 768 * 2, 121 +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + tiled=True, + use_two_stage_pipeline=True, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2_camera_jib_down.mp4', + fps=24, + audio_sample_rate=24000, +) diff --git a/examples/ltx2/model_inference_low_vram/LTX-2-T2AV-Camera-Control-Jib-Up.py b/examples/ltx2/model_inference_low_vram/LTX-2-T2AV-Camera-Control-Jib-Up.py new file mode 100644 index 0000000000000000000000000000000000000000..b6229c15f50b4cdc599b04448c7e69b01f30a3e4 --- /dev/null +++ b/examples/ltx2/model_inference_low_vram/LTX-2-T2AV-Camera-Control-Jib-Up.py @@ -0,0 +1,71 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 + +vram_config = { + "offload_dtype": torch.float8_e5m2, + "offload_device": "cpu", + "onload_dtype": torch.float8_e5m2, + "onload_device": "cpu", + "preparing_dtype": torch.float8_e5m2, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="transformer.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="text_encoder_post_modules.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vocoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_encoder.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-spatial-upscaler-x2-1.0.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), + stage2_lora_config=ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-19b-distilled-lora-384.safetensors"), +) +pipe.load_lora( + pipe.dit, + ModelConfig(model_id="Lightricks/LTX-2-19b-LoRA-Camera-Control-Jib-Up", origin_file_pattern="ltx-2-19b-lora-camera-control-jib-up.safetensors"), +) +prompt = ( + "A girl stands happily at a sleek desk with a glowing 'Diffsynth-Studio' logo, saying: “I enjoy working with Diffsynth-Studio, it's a perfect framework.” " + "The shot starts low—framing her waist, shoes, and a branded floor mat—and smoothly jib-ups upward. " + "As the camera rises, it reveals her smiling face, upper body, and behind her: a bright creative studio with wall art and a large window showing daylight sky. " + "The final frame fully shows the inspiring workspace above the initial view, ensuring spatial continuity." +) +negative_prompt = ( + "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, " + "grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, " + "deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, " + "wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of " + "field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent " + "lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny " + "valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, " + "mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, " + "off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward " + "pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, " + "inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +) +height, width, num_frames = 512 * 2, 768 * 2, 121 +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + tiled=True, + use_two_stage_pipeline=True, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2_camera_jib_up.mp4', + fps=24, + audio_sample_rate=24000, +) diff --git a/examples/ltx2/model_inference_low_vram/LTX-2-T2AV-Camera-Control-Static.py b/examples/ltx2/model_inference_low_vram/LTX-2-T2AV-Camera-Control-Static.py new file mode 100644 index 0000000000000000000000000000000000000000..13d8ea91b41e441504cd8c52be614cc2b9da1825 --- /dev/null +++ b/examples/ltx2/model_inference_low_vram/LTX-2-T2AV-Camera-Control-Static.py @@ -0,0 +1,66 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 + +vram_config = { + "offload_dtype": torch.float8_e5m2, + "offload_device": "cpu", + "onload_dtype": torch.float8_e5m2, + "onload_device": "cpu", + "preparing_dtype": torch.float8_e5m2, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="transformer.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="text_encoder_post_modules.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vocoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_encoder.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-spatial-upscaler-x2-1.0.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), + stage2_lora_config=ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-19b-distilled-lora-384.safetensors"), +) +pipe.load_lora( + pipe.dit, + ModelConfig(model_id="Lightricks/LTX-2-19b-LoRA-Camera-Control-Static", origin_file_pattern="ltx-2-19b-lora-camera-control-static.safetensors"), +) +prompt = "A beautiful sunset over the ocean." +negative_prompt = ( + "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, " + "grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, " + "deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, " + "wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of " + "field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent " + "lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny " + "valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, " + "mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, " + "off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward " + "pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, " + "inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +) +height, width, num_frames = 512 * 2, 768 * 2, 121 +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + tiled=True, + use_two_stage_pipeline=True, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2_camera_static.mp4', + fps=24, + audio_sample_rate=24000, +) diff --git a/examples/ltx2/model_inference_low_vram/LTX-2-T2AV-DistilledPipeline.py b/examples/ltx2/model_inference_low_vram/LTX-2-T2AV-DistilledPipeline.py new file mode 100644 index 0000000000000000000000000000000000000000..daf82f280b9798eae4bc6a0e09dbf16c950c0a02 --- /dev/null +++ b/examples/ltx2/model_inference_low_vram/LTX-2-T2AV-DistilledPipeline.py @@ -0,0 +1,63 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 + +vram_config = { + "offload_dtype": torch.float8_e5m2, + "offload_device": "cpu", + "onload_dtype": torch.float8_e5m2, + "onload_device": "cpu", + "preparing_dtype": torch.float8_e5m2, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="transformer_distilled.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="text_encoder_post_modules.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vocoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_encoder.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-spatial-upscaler-x2-1.0.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), + vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 0.5, +) + +prompt = "A girl is very happy, she is speaking: “I enjoy working with Diffsynth-Studio, it's a perfect framework.”" +negative_prompt = ( + "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, " + "grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, " + "deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, " + "wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of " + "field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent " + "lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny " + "valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, " + "mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, " + "off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward " + "pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, " + "inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +) +height, width, num_frames = 512 * 2, 768 * 2, 121 +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + tiled=True, + use_distilled_pipeline=True, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2_distilled.mp4', + fps=24, + audio_sample_rate=24000, +) diff --git a/examples/ltx2/model_inference_low_vram/LTX-2-T2AV-IC-LoRA-Detailer.py b/examples/ltx2/model_inference_low_vram/LTX-2-T2AV-IC-LoRA-Detailer.py new file mode 100644 index 0000000000000000000000000000000000000000..eccddd7c08a3d0bfffadaf257b56b6b8ac4c0fb6 --- /dev/null +++ b/examples/ltx2/model_inference_low_vram/LTX-2-T2AV-IC-LoRA-Detailer.py @@ -0,0 +1,77 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 +from diffsynth.utils.data import VideoData +from modelscope import dataset_snapshot_download + +vram_config = { + "offload_dtype": torch.float8_e5m2, + "offload_device": "cpu", + "onload_dtype": torch.float8_e5m2, + "onload_device": "cpu", + "preparing_dtype": torch.float8_e5m2, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="transformer.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="text_encoder_post_modules.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vocoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_encoder.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-spatial-upscaler-x2-1.0.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), + stage2_lora_config=ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-19b-distilled-lora-384.safetensors"), +) +pipe.load_lora(pipe.dit, ModelConfig(model_id="Lightricks/LTX-2-19b-IC-LoRA-Detailer", origin_file_pattern="ltx-2-19b-ic-lora-detailer.safetensors")) +dataset_snapshot_download("DiffSynth-Studio/example_video_dataset", allow_file_pattern="ltx2/*", local_dir="data/example_video_dataset") + +prompt = "[VISUAL]:Two cute orange cats, wearing boxing gloves, stand on a boxing ring and fight each other. [SOUNDS]:the sound of two cats boxing" +negative_prompt = ( + "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, " + "grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, " + "deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, " + "wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of " + "field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent " + "lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny " + "valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, " + "mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, " + "off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward " + "pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, " + "inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +) +height, width, num_frames = 512 * 2, 768 * 2, 121 +ref_scale_factor = 1 +frame_rate = 24 +# the frame rate of the video should better be the same with the reference video +# the spatial resolution of the first frame should be the resolution of stage 1 video generation divided by ref_scale_factor +input_video = VideoData("data/example_video_dataset/ltx2/video1.mp4", height=height // ref_scale_factor // 2, width=width // ref_scale_factor // 2) +input_video = input_video.raw_data() +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + frame_rate=frame_rate, + in_context_videos=[input_video], + in_context_downsample_factor=ref_scale_factor, + tiled=True, + use_two_stage_pipeline=True, + clear_lora_before_state_two=True, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2_twostage_iclora.mp4', + fps=frame_rate, + audio_sample_rate=24000, +) diff --git a/examples/ltx2/model_inference_low_vram/LTX-2-T2AV-IC-LoRA-Union-Control.py b/examples/ltx2/model_inference_low_vram/LTX-2-T2AV-IC-LoRA-Union-Control.py new file mode 100644 index 0000000000000000000000000000000000000000..37515d86c7df3b0abd5952b48c180d34803ba1fa --- /dev/null +++ b/examples/ltx2/model_inference_low_vram/LTX-2-T2AV-IC-LoRA-Union-Control.py @@ -0,0 +1,77 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 +from diffsynth.utils.data import VideoData +from modelscope import dataset_snapshot_download + +vram_config = { + "offload_dtype": torch.float8_e5m2, + "offload_device": "cpu", + "onload_dtype": torch.float8_e5m2, + "onload_device": "cpu", + "preparing_dtype": torch.float8_e5m2, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="transformer.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="text_encoder_post_modules.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vocoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_encoder.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-spatial-upscaler-x2-1.0.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), + stage2_lora_config=ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-19b-distilled-lora-384.safetensors"), +) +pipe.load_lora(pipe.dit, ModelConfig(model_id="Lightricks/LTX-2-19b-IC-LoRA-Union-Control", origin_file_pattern="ltx-2-19b-ic-lora-union-control-ref0.5.safetensors")) +dataset_snapshot_download("DiffSynth-Studio/example_video_dataset", allow_file_pattern="ltx2/*", local_dir="data/example_video_dataset") + +prompt = "[VISUAL]:Two cute orange cats, wearing boxing gloves, stand on a boxing ring and fight each other. [SOUNDS]:the sound of two cats boxing" +negative_prompt = ( + "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, " + "grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, " + "deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, " + "wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of " + "field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent " + "lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny " + "valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, " + "mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, " + "off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward " + "pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, " + "inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +) +height, width, num_frames = 512 * 2, 768 * 2, 121 +ref_scale_factor = 2 +frame_rate = 24 +# the frame rate of the video should better be the same with the reference video +# the spatial resolution of the first frame should be the resolution of stage 1 video generation divided by ref_scale_factor +input_video = VideoData("data/example_video_dataset/ltx2/depth_video.mp4", height=height // ref_scale_factor // 2, width=width // ref_scale_factor // 2) +input_video = input_video.raw_data() +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + frame_rate=frame_rate, + in_context_videos=[input_video], + in_context_downsample_factor=ref_scale_factor, + tiled=True, + use_two_stage_pipeline=True, + clear_lora_before_state_two=True, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2_twostage_iclora.mp4', + fps=frame_rate, + audio_sample_rate=24000, +) diff --git a/examples/ltx2/model_inference_low_vram/LTX-2-T2AV-OneStage.py b/examples/ltx2/model_inference_low_vram/LTX-2-T2AV-OneStage.py new file mode 100644 index 0000000000000000000000000000000000000000..c08332fba5c4fd12e1d7e0702448e28bf06434ac --- /dev/null +++ b/examples/ltx2/model_inference_low_vram/LTX-2-T2AV-OneStage.py @@ -0,0 +1,67 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 + +vram_config = { + "offload_dtype": torch.float8_e5m2, + "offload_device": "cpu", + "onload_dtype": torch.float8_e5m2, + "onload_device": "cpu", + "preparing_dtype": torch.float8_e5m2, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +""" +Offical model repo: https://www.modelscope.cn/models/Lightricks/LTX-2 +Repackaged model repo: https://www.modelscope.cn/models/DiffSynth-Studio/LTX-2-Repackage +For base models of LTX-2, offical checkpoint (with model config ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-19b-dev.safetensors")) +and repackaged checkpoints (with model config ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="*.safetensors")) are both supported. +We have repackeged the official checkpoints in DiffSynth-Studio/LTX-2-Repackage repo to support separate loading of different submodules, +and avoid redundant memory usage when users only want to use part of the model. +""" +# use the repackaged modelconfig from "DiffSynth-Studio/LTX-2-Repackage" to avoid redundant model loading +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="transformer.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="text_encoder_post_modules.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vocoder.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), + vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 0.5, +) +# use the following modelconfig if you want to initialize model from offical checkpoints from "Lightricks/LTX-2" +# pipe = LTX2AudioVideoPipeline.from_pretrained( +# torch_dtype=torch.bfloat16, +# device="cuda", +# model_configs=[ +# ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), +# ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-19b-dev.safetensors", **vram_config), +# ], +# tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), +# vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 0.5, +# ) +prompt = "A girl is very happy, she is speaking: “I enjoy working with Diffsynth-Studio, it's a perfect framework.”" +negative_prompt = "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +height, width, num_frames = 512, 768, 121 +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + tiled=True, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2_onestage.mp4', + fps=24, + audio_sample_rate=24000, +) diff --git a/examples/ltx2/model_inference_low_vram/LTX-2-T2AV-TwoStage.py b/examples/ltx2/model_inference_low_vram/LTX-2-T2AV-TwoStage.py new file mode 100644 index 0000000000000000000000000000000000000000..98ed96670de259d172b27d134f17ae2c7236093c --- /dev/null +++ b/examples/ltx2/model_inference_low_vram/LTX-2-T2AV-TwoStage.py @@ -0,0 +1,86 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 + +vram_config = { + "offload_dtype": torch.float8_e5m2, + "offload_device": "cpu", + "onload_dtype": torch.float8_e5m2, + "onload_device": "cpu", + "preparing_dtype": torch.float8_e5m2, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +""" +Offical model repo: https://www.modelscope.cn/models/Lightricks/LTX-2 +Repackaged model repo: https://www.modelscope.cn/models/DiffSynth-Studio/LTX-2-Repackage +For base models of LTX-2, offical checkpoint (with model config ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-19b-dev.safetensors")) +and repackaged checkpoints (with model config ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="*.safetensors")) are both supported. +We have repackeged the official checkpoints in DiffSynth-Studio/LTX-2-Repackage repo to support separate loading of different submodules, +and avoid redundant memory usage when users only want to use part of the model. +""" +# use the repackaged modelconfig from "DiffSynth-Studio/LTX-2-Repackage" to avoid redundant model loading +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="transformer.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="text_encoder_post_modules.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vocoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_encoder.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-spatial-upscaler-x2-1.0.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), + stage2_lora_config=ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-19b-distilled-lora-384.safetensors"), + vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 0.5, +) +# use the following modelconfig if you want to initialize model from offical checkpoints from "Lightricks/LTX-2" +# pipe = LTX2AudioVideoPipeline.from_pretrained( +# torch_dtype=torch.bfloat16, +# device="cuda", +# model_configs=[ +# ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), +# ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-19b-dev.safetensors", **vram_config), +# ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-spatial-upscaler-x2-1.0.safetensors", **vram_config), +# ], +# tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), +# stage2_lora_config=ModelConfig(model_id="Lightricks/LTX-2", origin_file_pattern="ltx-2-19b-distilled-lora-384.safetensors"), +# vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 0.5, +# ) + +prompt = "A girl is very happy, she is speaking: “I enjoy working with Diffsynth-Studio, it's a perfect framework.”" +negative_prompt = ( + "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, " + "grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, " + "deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, " + "wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of " + "field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent " + "lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny " + "valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, " + "mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, " + "off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward " + "pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, " + "inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +) +height, width, num_frames = 512 * 2, 768 * 2, 121 +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + tiled=True, + use_two_stage_pipeline=True, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2_twostage.mp4', + fps=24, + audio_sample_rate=24000, +) diff --git a/examples/ltx2/model_inference_low_vram/LTX-2.3-A2V-TwoStage.py b/examples/ltx2/model_inference_low_vram/LTX-2.3-A2V-TwoStage.py new file mode 100644 index 0000000000000000000000000000000000000000..022160bb9c6f1ff59329d7aaae33c80066b20d60 --- /dev/null +++ b/examples/ltx2/model_inference_low_vram/LTX-2.3-A2V-TwoStage.py @@ -0,0 +1,67 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 +from diffsynth.utils.data.audio import read_audio +from modelscope import dataset_snapshot_download + +vram_config = { + "offload_dtype": torch.float8_e5m2, + "offload_device": "cpu", + "onload_dtype": torch.float8_e5m2, + "onload_device": "cpu", + "preparing_dtype": torch.float8_e5m2, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2.3", origin_file_pattern="ltx-2.3-22b-dev.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2.3", origin_file_pattern="ltx-2.3-spatial-upscaler-x2-1.0.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), + stage2_lora_config=ModelConfig(model_id="Lightricks/LTX-2.3", origin_file_pattern="ltx-2.3-22b-distilled-lora-384.safetensors"), + vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 0.5, +) + +dataset_snapshot_download("DiffSynth-Studio/example_video_dataset", allow_file_pattern="ltx2/*", local_dir="data/example_video_dataset") +prompt = "A beautiful woman with a flower crown is singing happily under a blooming cherry tree." +negative_prompt = ( + "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, " + "grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, " + "deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, " + "wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of " + "field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent " + "lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny " + "valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, " + "mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, " + "off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward " + "pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, " + "inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +) +height, width, num_frames, frame_rate = 512 * 2, 768 * 2, 121, 24 +duration = num_frames / frame_rate +audio, audio_sample_rate = read_audio("data/example_video_dataset/ltx2/sing.MP3", start_time=1, duration=duration) +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + retake_audio=audio, + audio_sample_rate=audio_sample_rate, + seed=43, + height=height, + width=width, + num_frames=num_frames, + frame_rate=frame_rate, + tiled=True, + use_two_stage_pipeline=True, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2.3_twostage_a2v.mp4', + fps=frame_rate, + audio_sample_rate=pipe.audio_vocoder.output_sampling_rate, +) diff --git a/examples/ltx2/model_inference_low_vram/LTX-2.3-I2AV-DistilledPipeline.py b/examples/ltx2/model_inference_low_vram/LTX-2.3-I2AV-DistilledPipeline.py new file mode 100644 index 0000000000000000000000000000000000000000..5585efc676f5b20139ea43a660963f7b0035bd85 --- /dev/null +++ b/examples/ltx2/model_inference_low_vram/LTX-2.3-I2AV-DistilledPipeline.py @@ -0,0 +1,71 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 +from PIL import Image +from modelscope import dataset_snapshot_download + +vram_config = { + "offload_dtype": torch.float8_e5m2, + "offload_device": "cpu", + "onload_dtype": torch.float8_e5m2, + "onload_device": "cpu", + "preparing_dtype": torch.float8_e5m2, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2.3", origin_file_pattern="ltx-2.3-22b-distilled.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2.3", origin_file_pattern="ltx-2.3-spatial-upscaler-x2-1.0.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), + vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 0.5, +) + + +prompt = "A girl is very happy, she is speaking: “I enjoy working with Diffsynth-Studio, it's a perfect framework.”" +negative_prompt = ( + "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, " + "grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, " + "deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, " + "wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of " + "field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent " + "lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny " + "valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, " + "mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, " + "off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward " + "pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, " + "inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +) +height, width, num_frames = 512 * 2, 768 * 2, 121 +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/examples_in_diffsynth", + local_dir="./", + allow_file_pattern=["data/examples/ltx-2/first_frame.jpg"] +) +image = Image.open("data/examples/ltx-2/first_frame.jpg").convert("RGB").resize((width, height)) +# first frame +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + tiled=True, + use_distilled_pipeline=True, + input_images=[image], + input_images_indexes=[0], + input_images_strength=1.0, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2.3_distilled_i2av_first.mp4', + fps=24, + audio_sample_rate=pipe.audio_vocoder.output_sampling_rate, +) diff --git a/examples/ltx2/model_inference_low_vram/LTX-2.3-I2AV-OneStage.py b/examples/ltx2/model_inference_low_vram/LTX-2.3-I2AV-OneStage.py new file mode 100644 index 0000000000000000000000000000000000000000..61cd20e26eeec70987686f4ed8d88cdf47ad36fe --- /dev/null +++ b/examples/ltx2/model_inference_low_vram/LTX-2.3-I2AV-OneStage.py @@ -0,0 +1,56 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 +from PIL import Image +from modelscope import dataset_snapshot_download + +vram_config = { + "offload_dtype": torch.float8_e5m2, + "offload_device": "cpu", + "onload_dtype": torch.float8_e5m2, + "onload_device": "cpu", + "preparing_dtype": torch.float8_e5m2, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2.3", origin_file_pattern="ltx-2.3-22b-dev.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), + vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 0.5, +) + +prompt = "A girl is very happy, she is speaking: “I enjoy working with Diffsynth-Studio, it's a perfect framework.”" +negative_prompt = "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +height, width, num_frames = 512, 768, 121 +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/examples_in_diffsynth", + local_dir="./", + allow_file_pattern=["data/examples/ltx-2/first_frame.jpg"] +) +image = Image.open("data/examples/ltx-2/first_frame.jpg").convert("RGB").resize((width, height)) +# first frame +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + tiled=False, + input_images=[image], + input_images_indexes=[0], + input_images_strength=1.0, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2.3_onestage_i2av_first.mp4', + fps=24, + audio_sample_rate=pipe.audio_vocoder.output_sampling_rate, +) diff --git a/examples/ltx2/model_inference_low_vram/LTX-2.3-I2AV-TwoStage.py b/examples/ltx2/model_inference_low_vram/LTX-2.3-I2AV-TwoStage.py new file mode 100644 index 0000000000000000000000000000000000000000..d23bd619db7f223eef9fdc16df52c43b54734314 --- /dev/null +++ b/examples/ltx2/model_inference_low_vram/LTX-2.3-I2AV-TwoStage.py @@ -0,0 +1,71 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 +from PIL import Image +from modelscope import dataset_snapshot_download + +vram_config = { + "offload_dtype": torch.float8_e5m2, + "offload_device": "cpu", + "onload_dtype": torch.float8_e5m2, + "onload_device": "cpu", + "preparing_dtype": torch.float8_e5m2, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2.3", origin_file_pattern="ltx-2.3-22b-dev.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2.3", origin_file_pattern="ltx-2.3-spatial-upscaler-x2-1.0.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), + stage2_lora_config=ModelConfig(model_id="Lightricks/LTX-2.3", origin_file_pattern="ltx-2.3-22b-distilled-lora-384.safetensors"), + vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 0.5, +) + +prompt = "A girl is very happy, she is speaking: “I enjoy working with Diffsynth-Studio, it's a perfect framework.”" +negative_prompt = ( + "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, " + "grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, " + "deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, " + "wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of " + "field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent " + "lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny " + "valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, " + "mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, " + "off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward " + "pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, " + "inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +) +height, width, num_frames = 512 * 2, 768 * 2, 121 +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/examples_in_diffsynth", + local_dir="./", + allow_file_pattern=["data/examples/ltx-2/first_frame.jpg"] +) +image = Image.open("data/examples/ltx-2/first_frame.jpg").convert("RGB").resize((width, height)) +# first frame +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=42, + height=height, + width=width, + num_frames=num_frames, + tiled=True, + use_two_stage_pipeline=True, + input_images=[image], + input_images_indexes=[0], + input_images_strength=1.0, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2.3_twostage_i2av_first.mp4', + fps=24, + audio_sample_rate=pipe.audio_vocoder.output_sampling_rate, +) diff --git a/examples/ltx2/model_inference_low_vram/LTX-2.3-T2AV-DistilledPipeline.py b/examples/ltx2/model_inference_low_vram/LTX-2.3-T2AV-DistilledPipeline.py new file mode 100644 index 0000000000000000000000000000000000000000..8d67de9313835bddec66e2be8f16baed7f68c557 --- /dev/null +++ b/examples/ltx2/model_inference_low_vram/LTX-2.3-T2AV-DistilledPipeline.py @@ -0,0 +1,58 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 + +vram_config = { + "offload_dtype": torch.float8_e5m2, + "offload_device": "cpu", + "onload_dtype": torch.float8_e5m2, + "onload_device": "cpu", + "preparing_dtype": torch.float8_e5m2, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2.3", origin_file_pattern="ltx-2.3-22b-distilled.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2.3", origin_file_pattern="ltx-2.3-spatial-upscaler-x2-1.0.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), + vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 0.5, +) + +prompt = "A girl is very happy, she is speaking: “I enjoy working with Diffsynth-Studio, it's a perfect framework.”" +negative_prompt = ( + "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, " + "grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, " + "deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, " + "wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of " + "field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent " + "lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny " + "valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, " + "mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, " + "off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward " + "pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, " + "inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +) +height, width, num_frames = 512 * 2, 768 * 2, 121 +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + tiled=True, + use_distilled_pipeline=True, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2.3_distilled.mp4', + fps=24, + audio_sample_rate=pipe.audio_vocoder.output_sampling_rate, +) diff --git a/examples/ltx2/model_inference_low_vram/LTX-2.3-T2AV-IC-LoRA-Motion-Track-Control.py b/examples/ltx2/model_inference_low_vram/LTX-2.3-T2AV-IC-LoRA-Motion-Track-Control.py new file mode 100644 index 0000000000000000000000000000000000000000..094289a7a82ed5eea27075810ed76224e17a306d --- /dev/null +++ b/examples/ltx2/model_inference_low_vram/LTX-2.3-T2AV-IC-LoRA-Motion-Track-Control.py @@ -0,0 +1,72 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 +from modelscope import dataset_snapshot_download +from diffsynth.utils.data import VideoData + +vram_config = { + "offload_dtype": torch.float8_e5m2, + "offload_device": "cpu", + "onload_dtype": torch.float8_e5m2, + "onload_device": "cpu", + "preparing_dtype": torch.float8_e5m2, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2.3", origin_file_pattern="ltx-2.3-22b-dev.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2.3", origin_file_pattern="ltx-2.3-spatial-upscaler-x2-1.0.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), + stage2_lora_config=ModelConfig(model_id="Lightricks/LTX-2.3", origin_file_pattern="ltx-2.3-22b-distilled-lora-384.safetensors"), + vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 0.5, +) +pipe.load_lora(pipe.dit, ModelConfig(model_id="Lightricks/LTX-2.3-22b-IC-LoRA-Motion-Track-Control", origin_file_pattern="ltx-2.3-22b-ic-lora-motion-track-control-ref0.5.safetensors")) +dataset_snapshot_download("DiffSynth-Studio/example_video_dataset", allow_file_pattern="ltx2/*", local_dir="data/example_video_dataset") +prompt = "[VISUAL]:Two cute orange cats, wearing boxing gloves, stand on a boxing ring and fight each other. [SOUNDS]:the sound of two cats boxing" +negative_prompt = ( + "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, " + "grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, " + "deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, " + "wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of " + "field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent " + "lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny " + "valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, " + "mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, " + "off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward " + "pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, " + "inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +) +height, width, num_frames = 512 * 2, 768 * 2, 121 +ref_scale_factor = 2 +frame_rate = 24 +input_image = VideoData("data/example_video_dataset/ltx2/video1.mp4", height=height // ref_scale_factor // 2, width=width // ref_scale_factor // 2)[0] +input_video = VideoData("data/example_video_dataset/ltx2/spatial_tracker_v2.mp4", height=height // ref_scale_factor // 2, width=width // ref_scale_factor // 2).raw_data() +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + frame_rate=frame_rate, + in_context_videos=[input_video], + in_context_downsample_factor=ref_scale_factor, + input_images=[input_image], + input_images_indexes=[0], + tiled=True, + use_two_stage_pipeline=True, + clear_lora_before_state_two=True, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2.3_ic_lora.mp4', + fps=frame_rate, + audio_sample_rate=pipe.audio_vocoder.output_sampling_rate, +) diff --git a/examples/ltx2/model_inference_low_vram/LTX-2.3-T2AV-IC-LoRA-Union-Control.py b/examples/ltx2/model_inference_low_vram/LTX-2.3-T2AV-IC-LoRA-Union-Control.py new file mode 100644 index 0000000000000000000000000000000000000000..b015f06ca03740cf19ee8116ff1fbb889b6e7e17 --- /dev/null +++ b/examples/ltx2/model_inference_low_vram/LTX-2.3-T2AV-IC-LoRA-Union-Control.py @@ -0,0 +1,69 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 +from modelscope import dataset_snapshot_download +from diffsynth.utils.data import VideoData + +vram_config = { + "offload_dtype": torch.float8_e5m2, + "offload_device": "cpu", + "onload_dtype": torch.float8_e5m2, + "onload_device": "cpu", + "preparing_dtype": torch.float8_e5m2, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2.3", origin_file_pattern="ltx-2.3-22b-dev.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2.3", origin_file_pattern="ltx-2.3-spatial-upscaler-x2-1.0.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), + stage2_lora_config=ModelConfig(model_id="Lightricks/LTX-2.3", origin_file_pattern="ltx-2.3-22b-distilled-lora-384.safetensors"), + vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 0.5, +) +pipe.load_lora(pipe.dit, ModelConfig(model_id="Lightricks/LTX-2.3-22b-IC-LoRA-Union-Control", origin_file_pattern="ltx-2.3-22b-ic-lora-union-control-ref0.5.safetensors")) +dataset_snapshot_download("DiffSynth-Studio/example_video_dataset", allow_file_pattern="ltx2/*", local_dir="data/example_video_dataset") +prompt = "[VISUAL]:Two cute orange cats, wearing boxing gloves, stand on a boxing ring and fight each other. [SOUNDS]:the sound of two cats boxing" +negative_prompt = ( + "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, " + "grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, " + "deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, " + "wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of " + "field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent " + "lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny " + "valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, " + "mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, " + "off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward " + "pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, " + "inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +) +height, width, num_frames = 512 * 2, 768 * 2, 121 +ref_scale_factor = 2 +frame_rate = 24 +input_video = VideoData("data/example_video_dataset/ltx2/depth_video.mp4", height=height // ref_scale_factor // 2, width=width // ref_scale_factor // 2).raw_data() +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + frame_rate=frame_rate, + in_context_videos=[input_video], + in_context_downsample_factor=ref_scale_factor, + tiled=True, + use_two_stage_pipeline=True, + clear_lora_before_state_two=True, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2.3_ic_lora.mp4', + fps=frame_rate, + audio_sample_rate=pipe.audio_vocoder.output_sampling_rate, +) diff --git a/examples/ltx2/model_inference_low_vram/LTX-2.3-T2AV-OneStage.py b/examples/ltx2/model_inference_low_vram/LTX-2.3-T2AV-OneStage.py new file mode 100644 index 0000000000000000000000000000000000000000..95fa5cab58b99ad5a9a93f4b23ecbeb105ac56dc --- /dev/null +++ b/examples/ltx2/model_inference_low_vram/LTX-2.3-T2AV-OneStage.py @@ -0,0 +1,43 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 + +vram_config = { + "offload_dtype": torch.float8_e5m2, + "offload_device": "cpu", + "onload_dtype": torch.float8_e5m2, + "onload_device": "cpu", + "preparing_dtype": torch.float8_e5m2, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2.3", origin_file_pattern="ltx-2.3-22b-dev.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), + vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 0.5, +) +prompt = "A girl is very happy, she is speaking: “I enjoy working with Diffsynth-Studio, it's a perfect framework.”" +negative_prompt = "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +height, width, num_frames = 512, 768, 121 +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + tiled=True, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2.3_onestage.mp4', + fps=24, + audio_sample_rate=pipe.audio_vocoder.output_sampling_rate, +) diff --git a/examples/ltx2/model_inference_low_vram/LTX-2.3-T2AV-TwoStage-Retake.py b/examples/ltx2/model_inference_low_vram/LTX-2.3-T2AV-TwoStage-Retake.py new file mode 100644 index 0000000000000000000000000000000000000000..65a6ebfe59a0dcc69ec450f46fcef39b271e0a6e --- /dev/null +++ b/examples/ltx2/model_inference_low_vram/LTX-2.3-T2AV-TwoStage-Retake.py @@ -0,0 +1,78 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 +from diffsynth.utils.data.audio import read_audio +from modelscope import dataset_snapshot_download +from diffsynth.utils.data import VideoData + +vram_config = { + "offload_dtype": torch.float8_e5m2, + "offload_device": "cpu", + "onload_dtype": torch.float8_e5m2, + "onload_device": "cpu", + "preparing_dtype": torch.float8_e5m2, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2.3", origin_file_pattern="ltx-2.3-22b-dev.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2.3", origin_file_pattern="ltx-2.3-spatial-upscaler-x2-1.0.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), + stage2_lora_config=ModelConfig(model_id="Lightricks/LTX-2.3", origin_file_pattern="ltx-2.3-22b-distilled-lora-384.safetensors"), + vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 0.5, +) + +dataset_snapshot_download("DiffSynth-Studio/example_video_dataset", allow_file_pattern="ltx2/*", local_dir="data/example_video_dataset") +prompt = "A beautiful woman with a flower crown is singing happily under a blooming cherry tree. She sings: 'Mummy don't know daddy's getting hot. At the body shop'" +negative_prompt = ( + "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, " + "grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, " + "deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, " + "wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of " + "field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent " + "lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny " + "valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, " + "mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, " + "off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward " + "pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, " + "inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +) + +height, width, num_frames, frame_rate = 512 * 2, 768 * 2, 121, 24 +path = "data/example_video_dataset/ltx2/video2.mp4" +video = VideoData(path, height=height, width=width).raw_data()[:num_frames] +assert len(video) == num_frames, f"Input video has {len(video)} frames, but expected {num_frames} frames based on the specified num_frames argument." +duration = num_frames / frame_rate +audio, audio_sample_rate = read_audio(path) + +# Regenerate the video within time regions. You can specify different time regions for video frames and audio retake. +# retake regions are in seconds, and the example below retakes video frames in the time regions of [1s, 2s] and [3s, 4s], and retakes audio in the time regions of [0s, 1s] and [4s, 5s]. +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + retake_video=video, + retake_video_regions=[(1, 2), (3, 4)], + retake_audio=audio, + audio_sample_rate=audio_sample_rate, + retake_audio_regions=[(0, 1), (4, 5)], + seed=43, + height=height, + width=width, + num_frames=num_frames, + frame_rate=frame_rate, + tiled=True, + use_two_stage_pipeline=True, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2.3_twostage_retake.mp4', + fps=frame_rate, + audio_sample_rate=pipe.audio_vocoder.output_sampling_rate, +) diff --git a/examples/ltx2/model_inference_low_vram/LTX-2.3-T2AV-TwoStage.py b/examples/ltx2/model_inference_low_vram/LTX-2.3-T2AV-TwoStage.py new file mode 100644 index 0000000000000000000000000000000000000000..a954b40e81f306815dd629dae0e035f240ba9521 --- /dev/null +++ b/examples/ltx2/model_inference_low_vram/LTX-2.3-T2AV-TwoStage.py @@ -0,0 +1,58 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 + +vram_config = { + "offload_dtype": torch.float8_e5m2, + "offload_device": "cpu", + "onload_dtype": torch.float8_e5m2, + "onload_device": "cpu", + "preparing_dtype": torch.float8_e5m2, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2.3", origin_file_pattern="ltx-2.3-22b-dev.safetensors", **vram_config), + ModelConfig(model_id="Lightricks/LTX-2.3", origin_file_pattern="ltx-2.3-spatial-upscaler-x2-1.0.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), + stage2_lora_config=ModelConfig(model_id="Lightricks/LTX-2.3", origin_file_pattern="ltx-2.3-22b-distilled-lora-384.safetensors"), + vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 0.5, +) +prompt = "A girl is very happy, she is speaking: “I enjoy working with Diffsynth-Studio, it's a perfect framework.”" +negative_prompt = ( + "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, " + "grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, " + "deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, " + "wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of " + "field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent " + "lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny " + "valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, " + "mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, " + "off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward " + "pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, " + "inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +) +height, width, num_frames = 512 * 2, 768 * 2, 121 +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + tiled=True, + use_two_stage_pipeline=True, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2.3_twostage.mp4', + fps=24, + audio_sample_rate=pipe.audio_vocoder.output_sampling_rate, +) diff --git a/examples/ltx2/model_training/full/JoyAI-Echo-T2AV-splited.sh b/examples/ltx2/model_training/full/JoyAI-Echo-T2AV-splited.sh new file mode 100644 index 0000000000000000000000000000000000000000..05212e04c0f18b781d000c45d75fc37023b61a7e --- /dev/null +++ b/examples/ltx2/model_training/full/JoyAI-Echo-T2AV-splited.sh @@ -0,0 +1,37 @@ +modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "ltx2/LTX-2.3-T2AV-splited/*" --local_dir ./data/diffsynth_example_dataset + +# Splited Training +accelerate launch examples/ltx2/model_training/train.py \ + --dataset_base_path data/diffsynth_example_dataset/ltx2/LTX-2.3-T2AV-splited \ + --dataset_metadata_path data/diffsynth_example_dataset/ltx2/LTX-2.3-T2AV-splited/metadata.csv \ + --data_file_keys "video,input_audio" \ + --extra_inputs "input_audio" \ + --height 512 \ + --width 768 \ + --num_frames 121 \ + --dataset_repeat 1 \ + --model_id_with_origin_paths "jd-opensource/JoyAI-Echo:JoyAI-Echo-release.safetensors,google/gemma-3-12b-it-qat-q4_0-unquantized:model-*.safetensors" \ + --learning_rate 1e-5 \ + --num_epochs 5 \ + --remove_prefix_in_ckpt "pipe.dit." \ + --output_path "./models/train/JoyAI-Echo-T2AV-full-splited-cache" \ + --trainable_models "dit" \ + --use_gradient_checkpointing \ + --task "sft:data_process" + +accelerate launch --config_file examples/qwen_image/model_training/full/accelerate_config_zero2offload.yaml examples/ltx2/model_training/train.py \ + --dataset_base_path ./models/train/JoyAI-Echo-T2AV-full-splited-cache \ + --data_file_keys "video,input_audio" \ + --extra_inputs "input_audio" \ + --height 512 \ + --width 768 \ + --num_frames 121 \ + --dataset_repeat 100 \ + --model_id_with_origin_paths "jd-opensource/JoyAI-Echo:JoyAI-Echo-release.safetensors" \ + --learning_rate 1e-5 \ + --num_epochs 5 \ + --remove_prefix_in_ckpt "pipe.dit." \ + --output_path "./models/train/JoyAI-Echo-T2AV-full" \ + --trainable_models "dit" \ + --use_gradient_checkpointing \ + --task "sft:train" diff --git a/examples/ltx2/model_training/full/LTX-2-T2AV-splited.sh b/examples/ltx2/model_training/full/LTX-2-T2AV-splited.sh new file mode 100644 index 0000000000000000000000000000000000000000..47ed269337e276ece8b74ab82e099d2f34d574bc --- /dev/null +++ b/examples/ltx2/model_training/full/LTX-2-T2AV-splited.sh @@ -0,0 +1,37 @@ +modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "ltx2/LTX-2-T2AV-splited/*" --local_dir ./data/diffsynth_example_dataset + +# Splited Training +accelerate launch examples/ltx2/model_training/train.py \ + --dataset_base_path data/diffsynth_example_dataset/ltx2/LTX-2-T2AV-splited \ + --dataset_metadata_path data/diffsynth_example_dataset/ltx2/LTX-2-T2AV-splited/metadata.csv \ + --data_file_keys "video,input_audio" \ + --extra_inputs "input_audio" \ + --height 512 \ + --width 768 \ + --num_frames 121 \ + --dataset_repeat 1 \ + --model_id_with_origin_paths "DiffSynth-Studio/LTX-2-Repackage:text_encoder_post_modules.safetensors,DiffSynth-Studio/LTX-2-Repackage:video_vae_encoder.safetensors,DiffSynth-Studio/LTX-2-Repackage:audio_vae_encoder.safetensors,google/gemma-3-12b-it-qat-q4_0-unquantized:model-*.safetensors" \ + --learning_rate 1e-5 \ + --num_epochs 5 \ + --remove_prefix_in_ckpt "pipe.dit." \ + --output_path "./models/train/LTX2-T2AV-full-splited-cache" \ + --trainable_models "dit" \ + --use_gradient_checkpointing \ + --task "sft:data_process" + +accelerate launch --config_file examples/qwen_image/model_training/full/accelerate_config_zero2offload.yaml examples/ltx2/model_training/train.py \ + --dataset_base_path ./models/train/LTX2-T2AV-full-splited-cache \ + --data_file_keys "video,input_audio" \ + --extra_inputs "input_audio" \ + --height 512 \ + --width 768 \ + --num_frames 121 \ + --dataset_repeat 100 \ + --model_id_with_origin_paths "DiffSynth-Studio/LTX-2-Repackage:transformer.safetensors" \ + --learning_rate 1e-5 \ + --num_epochs 5 \ + --remove_prefix_in_ckpt "pipe.dit." \ + --output_path "./models/train/LTX2-T2AV-full" \ + --trainable_models "dit" \ + --use_gradient_checkpointing \ + --task "sft:train" diff --git a/examples/ltx2/model_training/full/LTX-2.3-I2AV-splited.sh b/examples/ltx2/model_training/full/LTX-2.3-I2AV-splited.sh new file mode 100644 index 0000000000000000000000000000000000000000..b688b2eb70d2e33cdde494322f45fbc4d64efb72 --- /dev/null +++ b/examples/ltx2/model_training/full/LTX-2.3-I2AV-splited.sh @@ -0,0 +1,37 @@ +modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "ltx2/LTX-2.3-I2AV-splited/*" --local_dir ./data/diffsynth_example_dataset + +# Splited Training +accelerate launch examples/ltx2/model_training/train.py \ + --dataset_base_path data/diffsynth_example_dataset/ltx2/LTX-2.3-I2AV-splited \ + --dataset_metadata_path data/diffsynth_example_dataset/ltx2/LTX-2.3-I2AV-splited/metadata.csv \ + --data_file_keys "video,input_audio" \ + --extra_inputs "input_audio,input_image" \ + --height 512 \ + --width 768 \ + --num_frames 121 \ + --dataset_repeat 1 \ + --model_id_with_origin_paths "DiffSynth-Studio/LTX-2.3-Repackage:text_encoder_post_modules.safetensors,DiffSynth-Studio/LTX-2.3-Repackage:video_vae_encoder.safetensors,DiffSynth-Studio/LTX-2.3-Repackage:audio_vae_encoder.safetensors,google/gemma-3-12b-it-qat-q4_0-unquantized:model-*.safetensors" \ + --learning_rate 1e-5 \ + --num_epochs 5 \ + --remove_prefix_in_ckpt "pipe.dit." \ + --output_path "./models/train/LTX2.3-I2AV-full-splited-cache" \ + --trainable_models "dit" \ + --use_gradient_checkpointing \ + --task "sft:data_process" + +accelerate launch --config_file examples/qwen_image/model_training/full/accelerate_config_zero2offload.yaml examples/ltx2/model_training/train.py \ + --dataset_base_path ./models/train/LTX2.3-I2AV-full-splited-cache \ + --data_file_keys "video,input_audio" \ + --extra_inputs "input_audio,input_image" \ + --height 512 \ + --width 768 \ + --num_frames 121 \ + --dataset_repeat 100 \ + --model_id_with_origin_paths "DiffSynth-Studio/LTX-2.3-Repackage:transformer.safetensors" \ + --learning_rate 1e-5 \ + --num_epochs 5 \ + --remove_prefix_in_ckpt "pipe.dit." \ + --output_path "./models/train/LTX2.3-I2AV-full" \ + --trainable_models "dit" \ + --use_gradient_checkpointing \ + --task "sft:train" diff --git a/examples/ltx2/model_training/full/LTX-2.3-T2AV-splited.sh b/examples/ltx2/model_training/full/LTX-2.3-T2AV-splited.sh new file mode 100644 index 0000000000000000000000000000000000000000..24028d70d1d5b270434857551450537d8cfca6a4 --- /dev/null +++ b/examples/ltx2/model_training/full/LTX-2.3-T2AV-splited.sh @@ -0,0 +1,37 @@ +modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "ltx2/LTX-2.3-T2AV-splited/*" --local_dir ./data/diffsynth_example_dataset + +# Splited Training +accelerate launch examples/ltx2/model_training/train.py \ + --dataset_base_path data/diffsynth_example_dataset/ltx2/LTX-2.3-T2AV-splited \ + --dataset_metadata_path data/diffsynth_example_dataset/ltx2/LTX-2.3-T2AV-splited/metadata.csv \ + --data_file_keys "video,input_audio" \ + --extra_inputs "input_audio" \ + --height 512 \ + --width 768 \ + --num_frames 121 \ + --dataset_repeat 1 \ + --model_id_with_origin_paths "DiffSynth-Studio/LTX-2.3-Repackage:text_encoder_post_modules.safetensors,DiffSynth-Studio/LTX-2.3-Repackage:video_vae_encoder.safetensors,DiffSynth-Studio/LTX-2.3-Repackage:audio_vae_encoder.safetensors,google/gemma-3-12b-it-qat-q4_0-unquantized:model-*.safetensors" \ + --learning_rate 1e-5 \ + --num_epochs 5 \ + --remove_prefix_in_ckpt "pipe.dit." \ + --output_path "./models/train/LTX2.3-T2AV-full-splited-cache" \ + --trainable_models "dit" \ + --use_gradient_checkpointing \ + --task "sft:data_process" + +accelerate launch --config_file examples/qwen_image/model_training/full/accelerate_config_zero2offload.yaml examples/ltx2/model_training/train.py \ + --dataset_base_path ./models/train/LTX2.3-T2AV-full-splited-cache \ + --data_file_keys "video,input_audio" \ + --extra_inputs "input_audio" \ + --height 512 \ + --width 768 \ + --num_frames 121 \ + --dataset_repeat 100 \ + --model_id_with_origin_paths "DiffSynth-Studio/LTX-2.3-Repackage:transformer.safetensors" \ + --learning_rate 1e-5 \ + --num_epochs 5 \ + --remove_prefix_in_ckpt "pipe.dit." \ + --output_path "./models/train/LTX2.3-T2AV-full" \ + --trainable_models "dit" \ + --use_gradient_checkpointing \ + --task "sft:train" diff --git a/examples/ltx2/model_training/lora/JoyAI-Echo-T2AV-splited.sh b/examples/ltx2/model_training/lora/JoyAI-Echo-T2AV-splited.sh new file mode 100644 index 0000000000000000000000000000000000000000..5c92b2722718cc1b5a432615a8f4795087fa59f5 --- /dev/null +++ b/examples/ltx2/model_training/lora/JoyAI-Echo-T2AV-splited.sh @@ -0,0 +1,41 @@ +modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "ltx2/LTX-2.3-T2AV-splited/*" --local_dir ./data/diffsynth_example_dataset + +# Splited Training +accelerate launch examples/ltx2/model_training/train.py \ + --dataset_base_path data/diffsynth_example_dataset/ltx2/LTX-2.3-T2AV-splited \ + --dataset_metadata_path data/diffsynth_example_dataset/ltx2/LTX-2.3-T2AV-splited/metadata.csv \ + --data_file_keys "video,input_audio" \ + --extra_inputs "input_audio" \ + --height 512 \ + --width 768 \ + --num_frames 121 \ + --dataset_repeat 1 \ + --model_id_with_origin_paths "jd-opensource/JoyAI-Echo:JoyAI-Echo-release.safetensors,google/gemma-3-12b-it-qat-q4_0-unquantized:model-*.safetensors" \ + --learning_rate 1e-4 \ + --num_epochs 5 \ + --remove_prefix_in_ckpt "pipe.dit." \ + --output_path "./models/train/JoyAI-Echo-T2AV_lora-splited-cache" \ + --lora_base_model "dit" \ + --lora_target_modules "to_k,to_q,to_v,to_out.0" \ + --lora_rank 32 \ + --use_gradient_checkpointing \ + --task "sft:data_process" + +accelerate launch examples/ltx2/model_training/train.py \ + --dataset_base_path ./models/train/JoyAI-Echo-T2AV_lora-splited-cache \ + --data_file_keys "video,input_audio" \ + --extra_inputs "input_audio" \ + --height 512 \ + --width 768 \ + --num_frames 121 \ + --dataset_repeat 100 \ + --model_id_with_origin_paths "jd-opensource/JoyAI-Echo:JoyAI-Echo-release.safetensors" \ + --learning_rate 1e-4 \ + --num_epochs 5 \ + --remove_prefix_in_ckpt "pipe.dit." \ + --output_path "./models/train/JoyAI-Echo-T2AV_lora" \ + --lora_base_model "dit" \ + --lora_target_modules "to_k,to_q,to_v,to_out.0" \ + --lora_rank 32 \ + --use_gradient_checkpointing \ + --task "sft:train" diff --git a/examples/ltx2/model_training/lora/LTX-2-T2AV-IC-LoRA-splited.sh b/examples/ltx2/model_training/lora/LTX-2-T2AV-IC-LoRA-splited.sh new file mode 100644 index 0000000000000000000000000000000000000000..fa4d23c662efe112d0f89ffe95595dd4acd17a19 --- /dev/null +++ b/examples/ltx2/model_training/lora/LTX-2-T2AV-IC-LoRA-splited.sh @@ -0,0 +1,41 @@ +modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "ltx2/LTX-2-T2AV-IC-LoRA-splited/*" --local_dir ./data/diffsynth_example_dataset + +# Splited Training +accelerate launch examples/ltx2/model_training/train.py \ + --dataset_base_path data/diffsynth_example_dataset/ltx2/LTX-2-T2AV-IC-LoRA-splited \ + --dataset_metadata_path data/diffsynth_example_dataset/ltx2/LTX-2-T2AV-IC-LoRA-splited/metadata.json \ + --data_file_keys "video,input_audio,in_context_videos" \ + --extra_inputs "input_audio,in_context_videos,in_context_downsample_factor,frame_rate" \ + --height 512 \ + --width 768 \ + --num_frames 81 \ + --dataset_repeat 1 \ + --model_id_with_origin_paths "DiffSynth-Studio/LTX-2-Repackage:text_encoder_post_modules.safetensors,DiffSynth-Studio/LTX-2-Repackage:video_vae_encoder.safetensors,DiffSynth-Studio/LTX-2-Repackage:audio_vae_encoder.safetensors,google/gemma-3-12b-it-qat-q4_0-unquantized:model-*.safetensors" \ + --learning_rate 1e-4 \ + --num_epochs 5 \ + --remove_prefix_in_ckpt "pipe.dit." \ + --output_path "./models/train/LTX2-T2AV-IC-LoRA-splited-cache" \ + --lora_base_model "dit" \ + --lora_target_modules "to_k,to_q,to_v,to_out.0" \ + --lora_rank 32 \ + --use_gradient_checkpointing \ + --task "sft:data_process" + +accelerate launch examples/ltx2/model_training/train.py \ + --dataset_base_path ./models/train/LTX2-T2AV-IC-LoRA-splited-cache \ + --data_file_keys "video,input_audio,in_context_videos" \ + --extra_inputs "input_audio,in_context_videos,in_context_downsample_factor,frame_rate" \ + --height 512 \ + --width 768 \ + --num_frames 81 \ + --dataset_repeat 100 \ + --model_id_with_origin_paths "DiffSynth-Studio/LTX-2-Repackage:transformer.safetensors" \ + --learning_rate 1e-4 \ + --num_epochs 5 \ + --remove_prefix_in_ckpt "pipe.dit." \ + --output_path "./models/train/LTX2-T2AV-IC-LoRA" \ + --lora_base_model "dit" \ + --lora_target_modules "to_k,to_q,to_v,to_out.0" \ + --lora_rank 32 \ + --use_gradient_checkpointing \ + --task "sft:train" diff --git a/examples/ltx2/model_training/lora/LTX-2-T2AV-noaudio.sh b/examples/ltx2/model_training/lora/LTX-2-T2AV-noaudio.sh new file mode 100644 index 0000000000000000000000000000000000000000..ca91f2640c45004da59036fd76fd325d89e5fcda --- /dev/null +++ b/examples/ltx2/model_training/lora/LTX-2-T2AV-noaudio.sh @@ -0,0 +1,58 @@ +modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "ltx2/LTX-2-T2AV-noaudio/*" --local_dir ./data/diffsynth_example_dataset + +# single stage training +# accelerate launch examples/ltx2/model_training/train.py \ +# --dataset_base_path data/diffsynth_example_dataset/ltx2/LTX-2-T2AV-noaudio \ +# --dataset_metadata_path data/diffsynth_example_dataset/ltx2/LTX-2-T2AV-noaudio/metadata.csv \ +# --height 256 \ +# --width 384 \ +# --num_frames 25\ +# --dataset_repeat 100 \ +# --model_id_with_origin_paths "DiffSynth-Studio/LTX-2-Repackage:transformer.safetensors,DiffSynth-Studio/LTX-2-Repackage:text_encoder_post_modules.safetensors,DiffSynth-Studio/LTX-2-Repackage:video_vae_encoder.safetensors,DiffSynth-Studio/LTX-2-Repackage:audio_vae_encoder.safetensors,google/gemma-3-12b-it-qat-q4_0-unquantized:model-*.safetensors" \ +# --learning_rate 1e-4 \ +# --num_epochs 5 \ +# --remove_prefix_in_ckpt "pipe.dit." \ +# --output_path "./models/train/LTX2-T2AV-noaudio_lora" \ +# --lora_base_model "dit" \ +# --lora_target_modules "to_k,to_q,to_v,to_out.0" \ +# --lora_rank 32 \ +# --use_gradient_checkpointing \ +# --find_unused_parameters + + +# Splited Training +accelerate launch examples/ltx2/model_training/train.py \ + --dataset_base_path data/diffsynth_example_dataset/ltx2/LTX-2-T2AV-noaudio \ + --dataset_metadata_path data/diffsynth_example_dataset/ltx2/LTX-2-T2AV-noaudio/metadata.csv \ + --height 512 \ + --width 768 \ + --num_frames 121\ + --dataset_repeat 1 \ + --model_id_with_origin_paths "DiffSynth-Studio/LTX-2-Repackage:text_encoder_post_modules.safetensors,DiffSynth-Studio/LTX-2-Repackage:video_vae_encoder.safetensors,DiffSynth-Studio/LTX-2-Repackage:audio_vae_encoder.safetensors,google/gemma-3-12b-it-qat-q4_0-unquantized:model-*.safetensors" \ + --learning_rate 1e-4 \ + --num_epochs 5 \ + --remove_prefix_in_ckpt "pipe.dit." \ + --output_path "./models/train/LTX2-T2AV-noaudio_lora-splited-cache" \ + --lora_base_model "dit" \ + --lora_target_modules "to_k,to_q,to_v,to_out.0" \ + --lora_rank 32 \ + --use_gradient_checkpointing \ + --task "sft:data_process" + + +accelerate launch examples/ltx2/model_training/train.py \ + --dataset_base_path ./models/train/LTX2-T2AV-noaudio_lora-splited-cache \ + --height 512 \ + --width 768 \ + --num_frames 121\ + --dataset_repeat 100 \ + --model_id_with_origin_paths "DiffSynth-Studio/LTX-2-Repackage:transformer.safetensors" \ + --learning_rate 1e-4 \ + --num_epochs 5 \ + --remove_prefix_in_ckpt "pipe.dit." \ + --output_path "./models/train/LTX2-T2AV-noaudio_lora" \ + --lora_base_model "dit" \ + --lora_target_modules "to_k,to_q,to_v,to_out.0" \ + --lora_rank 32 \ + --use_gradient_checkpointing \ + --task "sft:train" diff --git a/examples/ltx2/model_training/lora/LTX-2-T2AV-splited.sh b/examples/ltx2/model_training/lora/LTX-2-T2AV-splited.sh new file mode 100644 index 0000000000000000000000000000000000000000..eefc17b24b03c24df2f0ddec63d181cac5322133 --- /dev/null +++ b/examples/ltx2/model_training/lora/LTX-2-T2AV-splited.sh @@ -0,0 +1,62 @@ +modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "ltx2/LTX-2-T2AV-splited/*" --local_dir ./data/diffsynth_example_dataset + +# Single Stage Training not recommended for T2AV due to the large memory consumption. Please use the Splited Training instead. +# accelerate launch examples/ltx2/model_training/train.py \ +# --dataset_base_path data/diffsynth_example_dataset/ltx2/LTX-2-T2AV-splited \ +# --dataset_metadata_path data/diffsynth_example_dataset/ltx2/LTX-2-T2AV-splited/metadata.csv \ +# --data_file_keys "video,input_audio" \ +# --extra_inputs "input_audio" \ +# --height 256 \ +# --width 384 \ +# --num_frames 25\ +# --dataset_repeat 100 \ +# --model_id_with_origin_paths "DiffSynth-Studio/LTX-2-Repackage:transformer.safetensors,DiffSynth-Studio/LTX-2-Repackage:text_encoder_post_modules.safetensors,DiffSynth-Studio/LTX-2-Repackage:video_vae_encoder.safetensors,DiffSynth-Studio/LTX-2-Repackage:audio_vae_encoder.safetensors,google/gemma-3-12b-it-qat-q4_0-unquantized:model-*.safetensors" \ +# --learning_rate 1e-4 \ +# --num_epochs 5 \ +# --remove_prefix_in_ckpt "pipe.dit." \ +# --output_path "./models/train/LTX2-T2AV_lora" \ +# --lora_base_model "dit" \ +# --lora_target_modules "to_k,to_q,to_v,to_out.0" \ +# --lora_rank 32 \ +# --use_gradient_checkpointing \ +# --find_unused_parameters + +# Splited Training +accelerate launch examples/ltx2/model_training/train.py \ + --dataset_base_path data/diffsynth_example_dataset/ltx2/LTX-2-T2AV-splited \ + --dataset_metadata_path data/diffsynth_example_dataset/ltx2/LTX-2-T2AV-splited/metadata.csv \ + --data_file_keys "video,input_audio" \ + --extra_inputs "input_audio" \ + --height 512 \ + --width 768 \ + --num_frames 121 \ + --dataset_repeat 1 \ + --model_id_with_origin_paths "DiffSynth-Studio/LTX-2-Repackage:text_encoder_post_modules.safetensors,DiffSynth-Studio/LTX-2-Repackage:video_vae_encoder.safetensors,DiffSynth-Studio/LTX-2-Repackage:audio_vae_encoder.safetensors,google/gemma-3-12b-it-qat-q4_0-unquantized:model-*.safetensors" \ + --learning_rate 1e-4 \ + --num_epochs 5 \ + --remove_prefix_in_ckpt "pipe.dit." \ + --output_path "./models/train/LTX2-T2AV_lora-splited-cache" \ + --lora_base_model "dit" \ + --lora_target_modules "to_k,to_q,to_v,to_out.0" \ + --lora_rank 32 \ + --use_gradient_checkpointing \ + --task "sft:data_process" + +accelerate launch examples/ltx2/model_training/train.py \ + --dataset_base_path ./models/train/LTX2-T2AV_lora-splited-cache \ + --data_file_keys "video,input_audio" \ + --extra_inputs "input_audio" \ + --height 512 \ + --width 768 \ + --num_frames 121 \ + --dataset_repeat 100 \ + --model_id_with_origin_paths "DiffSynth-Studio/LTX-2-Repackage:transformer.safetensors" \ + --learning_rate 1e-4 \ + --num_epochs 5 \ + --remove_prefix_in_ckpt "pipe.dit." \ + --output_path "./models/train/LTX2-T2AV_lora" \ + --lora_base_model "dit" \ + --lora_target_modules "to_k,to_q,to_v,to_out.0" \ + --lora_rank 32 \ + --use_gradient_checkpointing \ + --task "sft:train" diff --git a/examples/ltx2/model_training/lora/LTX-2.3-I2AV-splited.sh b/examples/ltx2/model_training/lora/LTX-2.3-I2AV-splited.sh new file mode 100644 index 0000000000000000000000000000000000000000..35239b0889b6802414b8efe02b05da0c6a291cb6 --- /dev/null +++ b/examples/ltx2/model_training/lora/LTX-2.3-I2AV-splited.sh @@ -0,0 +1,41 @@ +modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "ltx2/LTX-2.3-I2AV-splited/*" --local_dir ./data/diffsynth_example_dataset + +# Splited Training +accelerate launch examples/ltx2/model_training/train.py \ + --dataset_base_path data/diffsynth_example_dataset/ltx2/LTX-2.3-I2AV-splited \ + --dataset_metadata_path data/diffsynth_example_dataset/ltx2/LTX-2.3-I2AV-splited/metadata.csv \ + --data_file_keys "video,input_audio" \ + --extra_inputs "input_audio,input_image" \ + --height 512 \ + --width 768 \ + --num_frames 121 \ + --dataset_repeat 1 \ + --model_id_with_origin_paths "DiffSynth-Studio/LTX-2.3-Repackage:text_encoder_post_modules.safetensors,DiffSynth-Studio/LTX-2.3-Repackage:video_vae_encoder.safetensors,DiffSynth-Studio/LTX-2.3-Repackage:audio_vae_encoder.safetensors,google/gemma-3-12b-it-qat-q4_0-unquantized:model-*.safetensors" \ + --learning_rate 1e-4 \ + --num_epochs 5 \ + --remove_prefix_in_ckpt "pipe.dit." \ + --output_path "./models/train/LTX2.3-I2AV_lora-splited-cache" \ + --lora_base_model "dit" \ + --lora_target_modules "to_k,to_q,to_v,to_out.0" \ + --lora_rank 32 \ + --use_gradient_checkpointing \ + --task "sft:data_process" + +accelerate launch examples/ltx2/model_training/train.py \ + --dataset_base_path ./models/train/LTX2.3-I2AV_lora-splited-cache \ + --data_file_keys "video,input_audio" \ + --extra_inputs "input_audio,input_image" \ + --height 512 \ + --width 768 \ + --num_frames 121 \ + --dataset_repeat 100 \ + --model_id_with_origin_paths "DiffSynth-Studio/LTX-2.3-Repackage:transformer.safetensors" \ + --learning_rate 1e-4 \ + --num_epochs 5 \ + --remove_prefix_in_ckpt "pipe.dit." \ + --output_path "./models/train/LTX2.3-I2AV_lora" \ + --lora_base_model "dit" \ + --lora_target_modules "to_k,to_q,to_v,to_out.0" \ + --lora_rank 32 \ + --use_gradient_checkpointing \ + --task "sft:train" diff --git a/examples/ltx2/model_training/lora/LTX-2.3-T2AV-IC-LoRA-splited.sh b/examples/ltx2/model_training/lora/LTX-2.3-T2AV-IC-LoRA-splited.sh new file mode 100644 index 0000000000000000000000000000000000000000..7cfb82ac7f2d437b60b18984e0653b271bb5d743 --- /dev/null +++ b/examples/ltx2/model_training/lora/LTX-2.3-T2AV-IC-LoRA-splited.sh @@ -0,0 +1,41 @@ +modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "ltx2/LTX-2.3-T2AV-IC-LoRA-splited/*" --local_dir ./data/diffsynth_example_dataset + +# Splited Training +accelerate launch examples/ltx2/model_training/train.py \ + --dataset_base_path data/diffsynth_example_dataset/ltx2/LTX-2.3-T2AV-IC-LoRA-splited \ + --dataset_metadata_path data/diffsynth_example_dataset/ltx2/LTX-2.3-T2AV-IC-LoRA-splited/metadata.json \ + --data_file_keys "video,input_audio,in_context_videos" \ + --extra_inputs "input_audio,in_context_videos,in_context_downsample_factor,frame_rate" \ + --height 512 \ + --width 768 \ + --num_frames 81 \ + --dataset_repeat 1 \ + --model_id_with_origin_paths "DiffSynth-Studio/LTX-2.3-Repackage:text_encoder_post_modules.safetensors,DiffSynth-Studio/LTX-2.3-Repackage:video_vae_encoder.safetensors,DiffSynth-Studio/LTX-2.3-Repackage:audio_vae_encoder.safetensors,google/gemma-3-12b-it-qat-q4_0-unquantized:model-*.safetensors" \ + --learning_rate 1e-4 \ + --num_epochs 5 \ + --remove_prefix_in_ckpt "pipe.dit." \ + --output_path "./models/train/LTX2.3-T2AV-IC-LoRA-splited-cache" \ + --lora_base_model "dit" \ + --lora_target_modules "to_k,to_q,to_v,to_out.0" \ + --lora_rank 32 \ + --use_gradient_checkpointing \ + --task "sft:data_process" + +accelerate launch examples/ltx2/model_training/train.py \ + --dataset_base_path ./models/train/LTX2.3-T2AV-IC-LoRA-splited-cache \ + --data_file_keys "video,input_audio,in_context_videos" \ + --extra_inputs "input_audio,in_context_videos,in_context_downsample_factor,frame_rate" \ + --height 512 \ + --width 768 \ + --num_frames 81 \ + --dataset_repeat 100 \ + --model_id_with_origin_paths "DiffSynth-Studio/LTX-2.3-Repackage:transformer.safetensors" \ + --learning_rate 1e-4 \ + --num_epochs 5 \ + --remove_prefix_in_ckpt "pipe.dit." \ + --output_path "./models/train/LTX2.3-T2AV-IC-LoRA" \ + --lora_base_model "dit" \ + --lora_target_modules "to_k,to_q,to_v,to_out.0" \ + --lora_rank 32 \ + --use_gradient_checkpointing \ + --task "sft:train" diff --git a/examples/ltx2/model_training/lora/LTX-2.3-T2AV-splited.sh b/examples/ltx2/model_training/lora/LTX-2.3-T2AV-splited.sh new file mode 100644 index 0000000000000000000000000000000000000000..ff13a9ded983b2005ee16774137076454f92c98c --- /dev/null +++ b/examples/ltx2/model_training/lora/LTX-2.3-T2AV-splited.sh @@ -0,0 +1,41 @@ +modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "ltx2/LTX-2.3-T2AV-splited/*" --local_dir ./data/diffsynth_example_dataset + +# Splited Training +accelerate launch examples/ltx2/model_training/train.py \ + --dataset_base_path data/diffsynth_example_dataset/ltx2/LTX-2.3-T2AV-splited \ + --dataset_metadata_path data/diffsynth_example_dataset/ltx2/LTX-2.3-T2AV-splited/metadata.csv \ + --data_file_keys "video,input_audio" \ + --extra_inputs "input_audio" \ + --height 512 \ + --width 768 \ + --num_frames 121 \ + --dataset_repeat 1 \ + --model_id_with_origin_paths "DiffSynth-Studio/LTX-2.3-Repackage:text_encoder_post_modules.safetensors,DiffSynth-Studio/LTX-2.3-Repackage:video_vae_encoder.safetensors,DiffSynth-Studio/LTX-2.3-Repackage:audio_vae_encoder.safetensors,google/gemma-3-12b-it-qat-q4_0-unquantized:model-*.safetensors" \ + --learning_rate 1e-4 \ + --num_epochs 5 \ + --remove_prefix_in_ckpt "pipe.dit." \ + --output_path "./models/train/LTX2.3-T2AV_lora-splited-cache" \ + --lora_base_model "dit" \ + --lora_target_modules "to_k,to_q,to_v,to_out.0" \ + --lora_rank 32 \ + --use_gradient_checkpointing \ + --task "sft:data_process" + +accelerate launch examples/ltx2/model_training/train.py \ + --dataset_base_path ./models/train/LTX2.3-T2AV_lora-splited-cache \ + --data_file_keys "video,input_audio" \ + --extra_inputs "input_audio" \ + --height 512 \ + --width 768 \ + --num_frames 121 \ + --dataset_repeat 100 \ + --model_id_with_origin_paths "DiffSynth-Studio/LTX-2.3-Repackage:transformer.safetensors" \ + --learning_rate 1e-4 \ + --num_epochs 5 \ + --remove_prefix_in_ckpt "pipe.dit." \ + --output_path "./models/train/LTX2.3-T2AV_lora" \ + --lora_base_model "dit" \ + --lora_target_modules "to_k,to_q,to_v,to_out.0" \ + --lora_rank 32 \ + --use_gradient_checkpointing \ + --task "sft:train" diff --git a/examples/ltx2/model_training/scripts/split_model_statedicts.py b/examples/ltx2/model_training/scripts/split_model_statedicts.py new file mode 100644 index 0000000000000000000000000000000000000000..1ca2ff3258bdf3aeaf98dce59351ecbc07cd8fdb --- /dev/null +++ b/examples/ltx2/model_training/scripts/split_model_statedicts.py @@ -0,0 +1,104 @@ +from safetensors.torch import save_file +from diffsynth import hash_state_dict_keys +from diffsynth.core import load_state_dict +from diffsynth.models.model_loader import ModelPool + +model_pool = ModelPool() +state_dict = load_state_dict("models/Lightricks/LTX-2/ltx-2-19b-dev.safetensors") + +dit_state_dict = {} +for name in state_dict: + if name.startswith("model.diffusion_model."): + new_name = name.replace("model.diffusion_model.", "") + if new_name.startswith("audio_embeddings_connector.") or new_name.startswith("video_embeddings_connector."): + continue + dit_state_dict[name] = state_dict[name] + +print(f"dit_state_dict keys hash: {hash_state_dict_keys(dit_state_dict)}") +save_file(dit_state_dict, "models/DiffSynth-Studio/LTX-2-Repackage/transformer.safetensors") +model_pool.auto_load_model( + "models/DiffSynth-Studio/LTX-2-Repackage/transformer.safetensors", +) + + +video_vae_encoder_state_dict = {} +for name in state_dict: + if name.startswith("vae.encoder."): + video_vae_encoder_state_dict[name] = state_dict[name] + elif name.startswith("vae.per_channel_statistics."): + video_vae_encoder_state_dict[name] = state_dict[name] + +save_file(video_vae_encoder_state_dict, "models/DiffSynth-Studio/LTX-2-Repackage/video_vae_encoder.safetensors") +print(f"video_vae_encoder keys hash: {hash_state_dict_keys(video_vae_encoder_state_dict)}") +model_pool.auto_load_model("models/DiffSynth-Studio/LTX-2-Repackage/video_vae_encoder.safetensors") + + +video_vae_decoder_state_dict = {} +for name in state_dict: + if name.startswith("vae.decoder."): + video_vae_decoder_state_dict[name] = state_dict[name] + elif name.startswith("vae.per_channel_statistics."): + video_vae_decoder_state_dict[name] = state_dict[name] +save_file(video_vae_decoder_state_dict, "models/DiffSynth-Studio/LTX-2-Repackage/video_vae_decoder.safetensors") +print(f"video_vae_decoder keys hash: {hash_state_dict_keys(video_vae_decoder_state_dict)}") +model_pool.auto_load_model("models/DiffSynth-Studio/LTX-2-Repackage/video_vae_decoder.safetensors") + + +audio_vae_decoder_state_dict = {} +for name in state_dict: + if name.startswith("audio_vae.decoder."): + audio_vae_decoder_state_dict[name] = state_dict[name] + elif name.startswith("audio_vae.per_channel_statistics."): + audio_vae_decoder_state_dict[name] = state_dict[name] +save_file(audio_vae_decoder_state_dict, "models/DiffSynth-Studio/LTX-2-Repackage/audio_vae_decoder.safetensors") +print(f"audio_vae_decoder keys hash: {hash_state_dict_keys(audio_vae_decoder_state_dict)}") +model_pool.auto_load_model("models/DiffSynth-Studio/LTX-2-Repackage/audio_vae_decoder.safetensors") + + +audio_vae_encoder_state_dict = {} +for name in state_dict: + if name.startswith("audio_vae.encoder."): + audio_vae_encoder_state_dict[name] = state_dict[name] + elif name.startswith("audio_vae.per_channel_statistics."): + audio_vae_encoder_state_dict[name] = state_dict[name] +save_file(audio_vae_encoder_state_dict, "models/DiffSynth-Studio/LTX-2-Repackage/audio_vae_encoder.safetensors") +print(f"audio_vae_encoder keys hash: {hash_state_dict_keys(audio_vae_encoder_state_dict)}") +model_pool.auto_load_model("models/DiffSynth-Studio/LTX-2-Repackage/audio_vae_encoder.safetensors") + + +audio_vocoder_state_dict = {} +for name in state_dict: + if name.startswith("vocoder."): + audio_vocoder_state_dict[name] = state_dict[name] +save_file(audio_vocoder_state_dict, "models/DiffSynth-Studio/LTX-2-Repackage/audio_vocoder.safetensors") +print(f"audio_vocoder keys hash: {hash_state_dict_keys(audio_vocoder_state_dict)}") +model_pool.auto_load_model("models/DiffSynth-Studio/LTX-2-Repackage/audio_vocoder.safetensors") + + +text_encoder_post_modules_state_dict = {} +for name in state_dict: + if name.startswith("text_embedding_projection."): + text_encoder_post_modules_state_dict[name] = state_dict[name] + elif name.startswith("model.diffusion_model.video_embeddings_connector."): + text_encoder_post_modules_state_dict[name] = state_dict[name] + elif name.startswith("model.diffusion_model.audio_embeddings_connector."): + text_encoder_post_modules_state_dict[name] = state_dict[name] +save_file(text_encoder_post_modules_state_dict, "models/DiffSynth-Studio/LTX-2-Repackage/text_encoder_post_modules.safetensors") +print(f"text_encoder_post_modules keys hash: {hash_state_dict_keys(text_encoder_post_modules_state_dict)}") +model_pool.auto_load_model("models/DiffSynth-Studio/LTX-2-Repackage/text_encoder_post_modules.safetensors") + + +state_dict = load_state_dict("models/Lightricks/LTX-2/ltx-2-19b-distilled.safetensors") +dit_state_dict = {} +for name in state_dict: + if name.startswith("model.diffusion_model."): + new_name = name.replace("model.diffusion_model.", "") + if new_name.startswith("audio_embeddings_connector.") or new_name.startswith("video_embeddings_connector."): + continue + dit_state_dict[name] = state_dict[name] + +print(f"dit_state_dict keys hash: {hash_state_dict_keys(dit_state_dict)}") +save_file(dit_state_dict, "models/DiffSynth-Studio/LTX-2-Repackage/transformer_distilled.safetensors") +model_pool.auto_load_model( + "models/DiffSynth-Studio/LTX-2-Repackage/transformer_distilled.safetensors", +) \ No newline at end of file diff --git a/examples/ltx2/model_training/scripts/split_model_statedicts_ltx2.3.py b/examples/ltx2/model_training/scripts/split_model_statedicts_ltx2.3.py new file mode 100644 index 0000000000000000000000000000000000000000..1dbc1bb3f6d5f1739a1d68b46f02eddd4403fb42 --- /dev/null +++ b/examples/ltx2/model_training/scripts/split_model_statedicts_ltx2.3.py @@ -0,0 +1,102 @@ +from safetensors.torch import save_file +from diffsynth import hash_state_dict_keys +from diffsynth.core import load_state_dict +from diffsynth.models.model_loader import ModelPool +import os + +model_pool = ModelPool() +state_dict = load_state_dict("models/Lightricks/LTX-2.3/ltx-2.3-22b-dev.safetensors") +os.makedirs("models/DiffSynth-Studio/LTX-2.3-Repackage", exist_ok=True) + +dit_state_dict = {} +for name in state_dict: + if name.startswith("model.diffusion_model."): + new_name = name.replace("model.diffusion_model.", "") + if new_name.startswith("audio_embeddings_connector.") or new_name.startswith("video_embeddings_connector."): + continue + dit_state_dict[name] = state_dict[name] + +print(f"dit_state_dict keys hash: {hash_state_dict_keys(dit_state_dict)}") +save_file(dit_state_dict, "models/DiffSynth-Studio/LTX-2.3-Repackage/transformer.safetensors") +model_pool.auto_load_model("models/DiffSynth-Studio/LTX-2.3-Repackage/transformer.safetensors") + + +video_vae_encoder_state_dict = {} +for name in state_dict: + if name.startswith("vae.encoder."): + video_vae_encoder_state_dict[name] = state_dict[name] + elif name.startswith("vae.per_channel_statistics."): + video_vae_encoder_state_dict[name] = state_dict[name] + +save_file(video_vae_encoder_state_dict, "models/DiffSynth-Studio/LTX-2.3-Repackage/video_vae_encoder.safetensors") +print(f"video_vae_encoder keys hash: {hash_state_dict_keys(video_vae_encoder_state_dict)}") +model_pool.auto_load_model("models/DiffSynth-Studio/LTX-2.3-Repackage/video_vae_encoder.safetensors") + + +video_vae_decoder_state_dict = {} +for name in state_dict: + if name.startswith("vae.decoder."): + video_vae_decoder_state_dict[name] = state_dict[name] + elif name.startswith("vae.per_channel_statistics."): + video_vae_decoder_state_dict[name] = state_dict[name] +save_file(video_vae_decoder_state_dict, "models/DiffSynth-Studio/LTX-2.3-Repackage/video_vae_decoder.safetensors") +print(f"video_vae_decoder keys hash: {hash_state_dict_keys(video_vae_decoder_state_dict)}") +model_pool.auto_load_model("models/DiffSynth-Studio/LTX-2.3-Repackage/video_vae_decoder.safetensors") + + +audio_vae_decoder_state_dict = {} +for name in state_dict: + if name.startswith("audio_vae.decoder."): + audio_vae_decoder_state_dict[name] = state_dict[name] + elif name.startswith("audio_vae.per_channel_statistics."): + audio_vae_decoder_state_dict[name] = state_dict[name] +save_file(audio_vae_decoder_state_dict, "models/DiffSynth-Studio/LTX-2.3-Repackage/audio_vae_decoder.safetensors") +print(f"audio_vae_decoder keys hash: {hash_state_dict_keys(audio_vae_decoder_state_dict)}") +model_pool.auto_load_model("models/DiffSynth-Studio/LTX-2.3-Repackage/audio_vae_decoder.safetensors") + + +audio_vae_encoder_state_dict = {} +for name in state_dict: + if name.startswith("audio_vae.encoder."): + audio_vae_encoder_state_dict[name] = state_dict[name] + elif name.startswith("audio_vae.per_channel_statistics."): + audio_vae_encoder_state_dict[name] = state_dict[name] +save_file(audio_vae_encoder_state_dict, "models/DiffSynth-Studio/LTX-2.3-Repackage/audio_vae_encoder.safetensors") +print(f"audio_vae_encoder keys hash: {hash_state_dict_keys(audio_vae_encoder_state_dict)}") +model_pool.auto_load_model("models/DiffSynth-Studio/LTX-2.3-Repackage/audio_vae_encoder.safetensors") + + +audio_vocoder_state_dict = {} +for name in state_dict: + if name.startswith("vocoder."): + audio_vocoder_state_dict[name] = state_dict[name] +save_file(audio_vocoder_state_dict, "models/DiffSynth-Studio/LTX-2.3-Repackage/audio_vocoder.safetensors") +print(f"audio_vocoder keys hash: {hash_state_dict_keys(audio_vocoder_state_dict)}") +model_pool.auto_load_model("models/DiffSynth-Studio/LTX-2.3-Repackage/audio_vocoder.safetensors") + + +text_encoder_post_modules_state_dict = {} +for name in state_dict: + if name.startswith("text_embedding_projection."): + text_encoder_post_modules_state_dict[name] = state_dict[name] + elif name.startswith("model.diffusion_model.video_embeddings_connector."): + text_encoder_post_modules_state_dict[name] = state_dict[name] + elif name.startswith("model.diffusion_model.audio_embeddings_connector."): + text_encoder_post_modules_state_dict[name] = state_dict[name] +save_file(text_encoder_post_modules_state_dict, "models/DiffSynth-Studio/LTX-2.3-Repackage/text_encoder_post_modules.safetensors") +print(f"text_encoder_post_modules keys hash: {hash_state_dict_keys(text_encoder_post_modules_state_dict)}") +model_pool.auto_load_model("models/DiffSynth-Studio/LTX-2.3-Repackage/text_encoder_post_modules.safetensors") + + +state_dict = load_state_dict("models/Lightricks/LTX-2.3/ltx-2.3-22b-distilled.safetensors") +dit_state_dict = {} +for name in state_dict: + if name.startswith("model.diffusion_model."): + new_name = name.replace("model.diffusion_model.", "") + if new_name.startswith("audio_embeddings_connector.") or new_name.startswith("video_embeddings_connector."): + continue + dit_state_dict[name] = state_dict[name] + +print(f"dit_state_dict keys hash: {hash_state_dict_keys(dit_state_dict)}") +save_file(dit_state_dict, "models/DiffSynth-Studio/LTX-2.3-Repackage/transformer_distilled.safetensors") +model_pool.auto_load_model("models/DiffSynth-Studio/LTX-2.3-Repackage/transformer_distilled.safetensors") diff --git a/examples/ltx2/model_training/special/split_training/LTX-2-T2AV-noaudio.sh b/examples/ltx2/model_training/special/split_training/LTX-2-T2AV-noaudio.sh new file mode 100644 index 0000000000000000000000000000000000000000..884e0e34efdffe030f0fbc532bfa2246f91f0ce5 --- /dev/null +++ b/examples/ltx2/model_training/special/split_training/LTX-2-T2AV-noaudio.sh @@ -0,0 +1,42 @@ +modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "ltx2/LTX-2-T2AV-noaudio/*" --local_dir ./data/diffsynth_example_dataset + +# Stage 1: cache deterministic preprocessing outputs. +accelerate launch examples/ltx2/model_training/train.py \ + --dataset_base_path data/diffsynth_example_dataset/ltx2/LTX-2-T2AV-noaudio \ + --dataset_metadata_path data/diffsynth_example_dataset/ltx2/LTX-2-T2AV-noaudio/metadata.csv \ + --height 256 \ + --width 384 \ + --num_frames 25 \ + --dataset_repeat 1 \ + --model_id_with_origin_paths 'DiffSynth-Studio/LTX-2-Repackage:transformer.safetensors,DiffSynth-Studio/LTX-2-Repackage:text_encoder_post_modules.safetensors,DiffSynth-Studio/LTX-2-Repackage:video_vae_encoder.safetensors,DiffSynth-Studio/LTX-2-Repackage:audio_vae_encoder.safetensors,google/gemma-3-12b-it-qat-q4_0-unquantized:model-*.safetensors' \ + --learning_rate 1e-4 \ + --num_epochs 5 \ + --remove_prefix_in_ckpt pipe.dit. \ + --output_path ./models/train/LTX2-T2AV-noaudio_split_cache \ + --lora_base_model dit \ + --lora_target_modules to_k,to_q,to_v,to_out.0 \ + --lora_rank 32 \ + --use_gradient_checkpointing \ + --find_unused_parameters \ + --offload_models DiffSynth-Studio/LTX-2-Repackage:transformer.safetensors \ + --task sft:data_process + +# Stage 2: train LoRA from the cached dataset. +accelerate launch examples/ltx2/model_training/train.py \ + --dataset_base_path ./models/train/LTX2-T2AV-noaudio_split_cache \ + --height 256 \ + --width 384 \ + --num_frames 25 \ + --dataset_repeat 100 \ + --model_id_with_origin_paths 'DiffSynth-Studio/LTX-2-Repackage:transformer.safetensors,DiffSynth-Studio/LTX-2-Repackage:text_encoder_post_modules.safetensors,DiffSynth-Studio/LTX-2-Repackage:video_vae_encoder.safetensors,DiffSynth-Studio/LTX-2-Repackage:audio_vae_encoder.safetensors,google/gemma-3-12b-it-qat-q4_0-unquantized:model-*.safetensors' \ + --learning_rate 1e-4 \ + --num_epochs 5 \ + --remove_prefix_in_ckpt pipe.dit. \ + --output_path ./models/train/LTX2-T2AV-noaudio_split \ + --lora_base_model dit \ + --lora_target_modules to_k,to_q,to_v,to_out.0 \ + --lora_rank 32 \ + --use_gradient_checkpointing \ + --find_unused_parameters \ + --offload_models 'DiffSynth-Studio/LTX-2-Repackage:text_encoder_post_modules.safetensors,DiffSynth-Studio/LTX-2-Repackage:video_vae_encoder.safetensors,DiffSynth-Studio/LTX-2-Repackage:audio_vae_encoder.safetensors,google/gemma-3-12b-it-qat-q4_0-unquantized:model-*.safetensors' \ + --task sft:train diff --git a/examples/ltx2/model_training/special/split_training/validate.py b/examples/ltx2/model_training/special/split_training/validate.py new file mode 100644 index 0000000000000000000000000000000000000000..77fcc98d4e76c4accf2d7bedebcb6c4934e4343d --- /dev/null +++ b/examples/ltx2/model_training/special/split_training/validate.py @@ -0,0 +1,48 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cuda", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="transformer.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="text_encoder_post_modules.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vocoder.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), +) +pipe.load_lora(pipe.dit, './models/train/LTX2-T2AV-noaudio_split/epoch-4.safetensors') +prompt = "A beautiful sunset over the ocean." +negative_prompt = "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +height, width, num_frames = 512, 768, 121 +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + tiled=True, + cfg_scale=4.0 +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='split_training_LTX-2-T2AV-noaudio.mp4', + fps=24, + audio_sample_rate=24000, +) diff --git a/examples/ltx2/model_training/train.py b/examples/ltx2/model_training/train.py new file mode 100644 index 0000000000000000000000000000000000000000..78be36affad5685af9a92d39fed09e4b7a5faa70 --- /dev/null +++ b/examples/ltx2/model_training/train.py @@ -0,0 +1,186 @@ +import torch, os, argparse, accelerate +from diffsynth.core import UnifiedDataset +from diffsynth.core.data.operators import LoadAudioWithTorchaudio, ToAbsolutePath, RouteByType, SequencialProcess +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.diffusion import * +os.environ["TOKENIZERS_PARALLELISM"] = "false" + + +class LTX2TrainingModule(DiffusionTrainingModule): + def __init__( + self, + model_paths=None, model_id_with_origin_paths=None, + tokenizer_path=None, + trainable_models=None, + lora_base_model=None, lora_target_modules="", lora_rank=32, lora_checkpoint=None, + preset_lora_path=None, preset_lora_model=None, + use_gradient_checkpointing=True, + use_gradient_checkpointing_offload=False, + extra_inputs=None, + fp8_models=None, + offload_models=None, + quant_options=None, + resume_from_checkpoint=None, remove_prefix_in_ckpt=None, + device="cpu", + task="sft", + ): + super().__init__() + # Load models + model_configs = self.parse_model_configs(model_paths, model_id_with_origin_paths, fp8_models=fp8_models, offload_models=offload_models, quant_options=quant_options, device=device) + tokenizer_config = ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized") if tokenizer_path is None else ModelConfig(tokenizer_path) + self.pipe = LTX2AudioVideoPipeline.from_pretrained(torch_dtype=torch.bfloat16, device=device, model_configs=model_configs, tokenizer_config=tokenizer_config) + self.pipe = self.split_pipeline_units( + task, self.pipe, trainable_models, lora_base_model, + remove_unnecessary_params=True, + force_remove_params_shared=("audio_latents", "video_latents"), + force_remove_params_nega=("audio_context", "video_context") + ) + self.resume_from_checkpoint(resume_from_checkpoint, remove_prefix_in_ckpt) + # Training mode + self.switch_pipe_to_training_mode( + self.pipe, trainable_models, + lora_base_model, lora_target_modules, lora_rank, lora_checkpoint, + preset_lora_path, preset_lora_model, + task=task, + ) + + # Store other configs + self.use_gradient_checkpointing = use_gradient_checkpointing + self.use_gradient_checkpointing_offload = use_gradient_checkpointing_offload + self.extra_inputs = extra_inputs.split(",") if extra_inputs is not None else [] + self.fp8_models = fp8_models + self.task = task + self.task_to_loss = { + "sft:data_process": lambda pipe, *args: args, + "sft": lambda pipe, inputs_shared, inputs_posi, inputs_nega: FlowMatchSFTAudioVideoLoss(pipe, **inputs_shared, **inputs_posi), + "sft:train": lambda pipe, inputs_shared, inputs_posi, inputs_nega: FlowMatchSFTAudioVideoLoss(pipe, **inputs_shared, **inputs_posi), + } + + def parse_extra_inputs(self, data, extra_inputs, inputs_shared): + for extra_input in extra_inputs: + if extra_input == "input_image": + inputs_shared["input_images"] = [data["video"][0]] + inputs_shared["input_images_indexes"] = [0] + inputs_shared["input_images_strength"] = 1.0 + else: + inputs_shared[extra_input] = data[extra_input] + return inputs_shared + + def get_pipeline_inputs(self, data): + inputs_posi = {"prompt": data["prompt"]} + inputs_nega = {} + inputs_shared = { + # Assume you are using this pipeline for inference, + # please fill in the input parameters. + "input_video": data["video"], + "height": data["video"][0].size[1], + "width": data["video"][0].size[0], + "num_frames": len(data["video"]), + "frame_rate": data.get("frame_rate", 24), + # Please do not modify the following parameters + # unless you clearly know what this will cause. + "cfg_scale": 1, + "tiled": False, + "rand_device": self.pipe.device, + "use_gradient_checkpointing": self.use_gradient_checkpointing, + "use_gradient_checkpointing_offload": self.use_gradient_checkpointing_offload, + "video_patchifier": self.pipe.video_patchifier, + "audio_patchifier": self.pipe.audio_patchifier, + } + inputs_shared = self.parse_extra_inputs(data, self.extra_inputs, inputs_shared) + return inputs_shared, inputs_posi, inputs_nega + + def forward(self, data, inputs=None): + if inputs is None: inputs = self.get_pipeline_inputs(data) + inputs = self.transfer_data_to_device(inputs, self.pipe.device, self.pipe.torch_dtype) + for unit in self.pipe.units: + inputs = self.pipe.unit_runner(unit, self.pipe, *inputs) + loss = self.task_to_loss[self.task](self.pipe, *inputs) + return loss + + +def ltx2_parser(): + parser = argparse.ArgumentParser(description="Simple example of a training script.") + parser = add_general_config(parser) + parser = add_video_size_config(parser) + parser.add_argument("--tokenizer_path", type=str, default=None, help="Path to tokenizer.") + parser.add_argument("--frame_rate", type=float, default=24, help="frame rate of the training videos.") + parser.add_argument("--initialize_model_on_cpu", default=False, action="store_true", help="Whether to initialize models on CPU.") + return parser + + +if __name__ == "__main__": + parser = ltx2_parser() + args = parser.parse_args() + accelerator = accelerate.Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + kwargs_handlers=[accelerate.DistributedDataParallelKwargs(find_unused_parameters=args.find_unused_parameters)], + ) + video_processor = UnifiedDataset.default_video_operator( + base_path=args.dataset_base_path, + max_pixels=args.max_pixels, + height=args.height, + width=args.width, + height_division_factor=32, + width_division_factor=32, + num_frames=args.num_frames, + time_division_factor=8, + time_division_remainder=1, + frame_rate=args.frame_rate, + fix_frame_rate=True, + ) + dataset = UnifiedDataset( + base_path=args.dataset_base_path, + metadata_path=args.dataset_metadata_path, + repeat=args.dataset_repeat, + data_file_keys=args.data_file_keys.split(","), + main_data_operator=video_processor, + special_operator_map={ + "input_audio": ToAbsolutePath(args.dataset_base_path) >> LoadAudioWithTorchaudio(num_frames=args.num_frames, time_division_factor=8, time_division_remainder=1, frame_rate=args.frame_rate), + "in_context_videos": RouteByType(operator_map=[ + (str, video_processor), + (list, SequencialProcess(video_processor)), + ]), + } + ) + model = LTX2TrainingModule( + model_paths=args.model_paths, + model_id_with_origin_paths=args.model_id_with_origin_paths, + tokenizer_path=args.tokenizer_path, + trainable_models=args.trainable_models, + lora_base_model=args.lora_base_model, + lora_target_modules=args.lora_target_modules, + lora_rank=args.lora_rank, + lora_checkpoint=args.lora_checkpoint, + preset_lora_path=args.preset_lora_path, + preset_lora_model=args.preset_lora_model, + use_gradient_checkpointing=args.use_gradient_checkpointing, + use_gradient_checkpointing_offload=args.use_gradient_checkpointing_offload, + extra_inputs=args.extra_inputs, + fp8_models=args.fp8_models, + offload_models=args.offload_models, + quant_options=args.quant_options, + resume_from_checkpoint=args.resume_from_checkpoint, + remove_prefix_in_ckpt=args.remove_prefix_in_ckpt, + task=args.task, + device="cpu" if (args.initialize_model_on_cpu or args.enable_model_cpu_offload) else accelerator.device, + ) + model_logger = ModelLogger( + args.output_path, + remove_prefix_in_ckpt=args.remove_prefix_in_ckpt, + enable_tensorboard_log=args.enable_tensorboard_log, + enable_swanlab_log=args.enable_swanlab_log, + swanlab_project=args.swanlab_project, + enable_wandb_log=args.enable_wandb_log, + wandb_project=args.wandb_project, + enable_csv_log=args.enable_csv_log, + ) + launcher_map = { + "sft:data_process": launch_data_process_task, + "direct_distill:data_process": launch_data_process_task, + "sft": launch_training_task, + "sft:train": launch_training_task, + "direct_distill": launch_training_task, + "direct_distill:train": launch_training_task, + } + launcher_map[args.task](accelerator, dataset, model, model_logger, args=args) diff --git a/examples/ltx2/model_training/validate_full/JoyAI-Echo-T2AV.py b/examples/ltx2/model_training/validate_full/JoyAI-Echo-T2AV.py new file mode 100644 index 0000000000000000000000000000000000000000..f9589324b5ac774fbc2746026d5300ba3bbd8d4a --- /dev/null +++ b/examples/ltx2/model_training/validate_full/JoyAI-Echo-T2AV.py @@ -0,0 +1,51 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 +from diffsynth import load_state_dict + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cuda", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} + +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="jd-opensource/JoyAI-Echo", origin_file_pattern="JoyAI-Echo-release.safetensors", **vram_config), + + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), +) + +state_dict = load_state_dict("models/train/JoyAI-Echo-T2AV-full/epoch-4.safetensors") +pipe.dit.load_state_dict(state_dict) + +prompt = "A beautiful sunset over the ocean." + +negative_prompt = "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +height, width, num_frames = 512, 768, 121 +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + use_distilled_pipeline=True, + tiled=True, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='joyai_echo.mp4', + fps=24, + audio_sample_rate=pipe.audio_vocoder.output_sampling_rate, +) diff --git a/examples/ltx2/model_training/validate_full/LTX-2-T2AV.py b/examples/ltx2/model_training/validate_full/LTX-2-T2AV.py new file mode 100644 index 0000000000000000000000000000000000000000..6201ec138a111177e7fae61e1220cc39af49ab20 --- /dev/null +++ b/examples/ltx2/model_training/validate_full/LTX-2-T2AV.py @@ -0,0 +1,47 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cuda", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="text_encoder_post_modules.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vocoder.safetensors", **vram_config), + ModelConfig(path="./models/train/LTX2-T2AV-full/epoch-4.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), +) +prompt = "A beautiful sunset over the ocean." +negative_prompt = "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +height, width, num_frames = 512, 768, 121 +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + tiled=True, + cfg_scale=4.0 +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2_onestage.mp4', + fps=24, + audio_sample_rate=24000, +) diff --git a/examples/ltx2/model_training/validate_full/LTX-2.3-I2AV.py b/examples/ltx2/model_training/validate_full/LTX-2.3-I2AV.py new file mode 100644 index 0000000000000000000000000000000000000000..f375ee60736d30dc37ef00a9328f3885d831b8b1 --- /dev/null +++ b/examples/ltx2/model_training/validate_full/LTX-2.3-I2AV.py @@ -0,0 +1,54 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 +from diffsynth.utils.data import VideoData + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cuda", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2.3-Repackage", origin_file_pattern="text_encoder_post_modules.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2.3-Repackage", origin_file_pattern="video_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2.3-Repackage", origin_file_pattern="audio_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2.3-Repackage", origin_file_pattern="audio_vocoder.safetensors", **vram_config), + ModelConfig(path="./models/train/LTX2.3-I2AV-full/epoch-4.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), +) + +prompt = "A beautiful sunset over the ocean." +negative_prompt = "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +height, width, num_frames = 512, 768, 121 +image = VideoData("data/example_video_dataset/ltx2/video.mp4", height=height, width=width)[0] +# first frame +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + tiled=False, + input_images=[image], + input_images_indexes=[0], + input_images_strength=1.0, + num_inference_steps=40, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2.3_onestage_i2av_first.mp4', + fps=24, + audio_sample_rate=pipe.audio_vocoder.output_sampling_rate, +) diff --git a/examples/ltx2/model_training/validate_full/LTX-2.3-T2AV.py b/examples/ltx2/model_training/validate_full/LTX-2.3-T2AV.py new file mode 100644 index 0000000000000000000000000000000000000000..d8392546ee5000368d992101e1e6c71fabcf7536 --- /dev/null +++ b/examples/ltx2/model_training/validate_full/LTX-2.3-T2AV.py @@ -0,0 +1,47 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cuda", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2.3-Repackage", origin_file_pattern="text_encoder_post_modules.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2.3-Repackage", origin_file_pattern="video_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2.3-Repackage", origin_file_pattern="audio_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2.3-Repackage", origin_file_pattern="audio_vocoder.safetensors", **vram_config), + ModelConfig(path="./models/train/LTX2.3-T2AV-full/epoch-4.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), +) +prompt = "A beautiful sunset over the ocean." +negative_prompt = "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +height, width, num_frames = 512, 768, 121 +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + tiled=True, + cfg_scale=4.0 +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2_onestage.mp4', + fps=24, + audio_sample_rate=24000, +) diff --git a/examples/ltx2/model_training/validate_lora/JoyAI-Echo-T2AV.py b/examples/ltx2/model_training/validate_lora/JoyAI-Echo-T2AV.py new file mode 100644 index 0000000000000000000000000000000000000000..4f5296959efae63a25eae9900f93ba09ab8c9c06 --- /dev/null +++ b/examples/ltx2/model_training/validate_lora/JoyAI-Echo-T2AV.py @@ -0,0 +1,49 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cuda", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} + +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="jd-opensource/JoyAI-Echo", origin_file_pattern="JoyAI-Echo-release.safetensors", **vram_config), + + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), +) + +pipe.load_lora(pipe.dit, "models/train/JoyAI-Echo-T2AV_lora/epoch-4.safetensors") + +prompt = "A beautiful sunset over the ocean." + +negative_prompt = "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +height, width, num_frames = 512, 768, 121 +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + use_distilled_pipeline=True, + tiled=True, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='joyai_echo.mp4', + fps=24, + audio_sample_rate=pipe.audio_vocoder.output_sampling_rate, +) diff --git a/examples/ltx2/model_training/validate_lora/LTX-2-T2AV-IC-LoRA.py b/examples/ltx2/model_training/validate_lora/LTX-2-T2AV-IC-LoRA.py new file mode 100644 index 0000000000000000000000000000000000000000..d6eda1a9605760c391e8addb41e29976dcabd0c8 --- /dev/null +++ b/examples/ltx2/model_training/validate_lora/LTX-2-T2AV-IC-LoRA.py @@ -0,0 +1,56 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 +from diffsynth.utils.data import VideoData + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cuda", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="transformer.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="text_encoder_post_modules.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vocoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_encoder.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), +) +pipe.load_lora(pipe.dit, "./models/train/LTX2-T2AV-IC-LoRA/epoch-4.safetensors") +prompt = "[VISUAL]:Two cute orange cats, wearing boxing gloves, stand on a boxing ring and fight each other. [SOUNDS]:the sound of two cats boxing" +negative_prompt = "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +height, width, num_frames = 512, 768, 81 +ref_scale_factor = 2 +frame_rate = 24 +input_video = VideoData("data/example_video_dataset/ltx2/depth_video.mp4", height=height // ref_scale_factor // 2, width=width // ref_scale_factor // 2) +input_video = input_video.raw_data() +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + frame_rate=frame_rate, + tiled=True, + in_context_videos=[input_video], + in_context_downsample_factor=ref_scale_factor, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2_onestage_ic.mp4', + fps=frame_rate, + audio_sample_rate=24000, +) diff --git a/examples/ltx2/model_training/validate_lora/LTX-2-T2AV.py b/examples/ltx2/model_training/validate_lora/LTX-2-T2AV.py new file mode 100644 index 0000000000000000000000000000000000000000..d0dab817d289522f94af0a3dc957c50305908419 --- /dev/null +++ b/examples/ltx2/model_training/validate_lora/LTX-2-T2AV.py @@ -0,0 +1,48 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cuda", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="transformer.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="text_encoder_post_modules.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vocoder.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), +) +pipe.load_lora(pipe.dit, "models/train/LTX2-T2AV_lora/epoch-4.safetensors") +prompt = "A beautiful sunset over the ocean." +negative_prompt = "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +height, width, num_frames = 512, 768, 121 +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + tiled=True, + cfg_scale=4.0 +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2_onestage.mp4', + fps=24, + audio_sample_rate=24000, +) diff --git a/examples/ltx2/model_training/validate_lora/LTX-2-T2AV_noaudio.py b/examples/ltx2/model_training/validate_lora/LTX-2-T2AV_noaudio.py new file mode 100644 index 0000000000000000000000000000000000000000..336b2bf5b978eacd4c8e41668dd38649936c7620 --- /dev/null +++ b/examples/ltx2/model_training/validate_lora/LTX-2-T2AV_noaudio.py @@ -0,0 +1,48 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cuda", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="transformer.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="text_encoder_post_modules.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="video_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2-Repackage", origin_file_pattern="audio_vocoder.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), +) +pipe.load_lora(pipe.dit, "models/train/LTX2-T2AV-noaudio_lora/epoch-4.safetensors") +prompt = "A beautiful sunset over the ocean." +negative_prompt = "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +height, width, num_frames = 512, 768, 121 +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + tiled=True, + cfg_scale=4.0 +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2_onestage.mp4', + fps=24, + audio_sample_rate=24000, +) diff --git a/examples/ltx2/model_training/validate_lora/LTX-2.3-I2AV.py b/examples/ltx2/model_training/validate_lora/LTX-2.3-I2AV.py new file mode 100644 index 0000000000000000000000000000000000000000..dc4093076b1dbdce4bb498c081f96cc73a43475f --- /dev/null +++ b/examples/ltx2/model_training/validate_lora/LTX-2.3-I2AV.py @@ -0,0 +1,56 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 +from diffsynth.utils.data import VideoData + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cuda", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2.3-Repackage", origin_file_pattern="transformer.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2.3-Repackage", origin_file_pattern="text_encoder_post_modules.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2.3-Repackage", origin_file_pattern="video_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2.3-Repackage", origin_file_pattern="audio_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2.3-Repackage", origin_file_pattern="audio_vocoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2.3-Repackage", origin_file_pattern="video_vae_encoder.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), +) +pipe.load_lora(pipe.dit, "models/train/LTX2.3-I2AV_lora/epoch-4.safetensors") + +prompt = "A beautiful sunset over the ocean." +negative_prompt = "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +height, width, num_frames = 512, 768, 121 +image = VideoData("data/example_video_dataset/ltx2/video.mp4", height=height, width=width)[0] +# first frame +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + tiled=False, + input_images=[image], + input_images_indexes=[0], + input_images_strength=1.0, + num_inference_steps=40, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2.3_onestage_i2av_first.mp4', + fps=24, + audio_sample_rate=pipe.audio_vocoder.output_sampling_rate, +) diff --git a/examples/ltx2/model_training/validate_lora/LTX-2.3-T2AV-IC-LoRA.py b/examples/ltx2/model_training/validate_lora/LTX-2.3-T2AV-IC-LoRA.py new file mode 100644 index 0000000000000000000000000000000000000000..4ade9a72df5aace48da3a30ce04b71ecd3e9b8af --- /dev/null +++ b/examples/ltx2/model_training/validate_lora/LTX-2.3-T2AV-IC-LoRA.py @@ -0,0 +1,56 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 +from diffsynth.utils.data import VideoData + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cuda", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2.3-Repackage", origin_file_pattern="transformer.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2.3-Repackage", origin_file_pattern="text_encoder_post_modules.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2.3-Repackage", origin_file_pattern="video_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2.3-Repackage", origin_file_pattern="audio_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2.3-Repackage", origin_file_pattern="audio_vocoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2.3-Repackage", origin_file_pattern="video_vae_encoder.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), +) +pipe.load_lora(pipe.dit, "./models/train/LTX2.3-T2AV-IC-LoRA/epoch-4.safetensors") +prompt = "[VISUAL]:Two cute orange cats, wearing boxing gloves, stand on a boxing ring and fight each other. [SOUNDS]:the sound of two cats boxing" +negative_prompt = "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +height, width, num_frames = 512, 768, 81 +ref_scale_factor = 2 +frame_rate = 24 +input_video = VideoData("data/example_video_dataset/ltx2/depth_video.mp4", height=height // ref_scale_factor // 2, width=width // ref_scale_factor // 2) +input_video = input_video.raw_data() +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + frame_rate=frame_rate, + tiled=True, + in_context_videos=[input_video], + in_context_downsample_factor=ref_scale_factor, +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2.3_onestage_ic.mp4', + fps=frame_rate, + audio_sample_rate=pipe.audio_vocoder.output_sampling_rate, +) diff --git a/examples/ltx2/model_training/validate_lora/LTX-2.3-T2AV.py b/examples/ltx2/model_training/validate_lora/LTX-2.3-T2AV.py new file mode 100644 index 0000000000000000000000000000000000000000..03d974b1ec4ac486b9501da7926dcf5844619031 --- /dev/null +++ b/examples/ltx2/model_training/validate_lora/LTX-2.3-T2AV.py @@ -0,0 +1,48 @@ +import torch +from diffsynth.pipelines.ltx2_audio_video import LTX2AudioVideoPipeline, ModelConfig +from diffsynth.utils.data.media_io_ltx2 import write_video_audio_ltx2 + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cuda", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = LTX2AudioVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized", origin_file_pattern="model-*.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2.3-Repackage", origin_file_pattern="transformer.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2.3-Repackage", origin_file_pattern="text_encoder_post_modules.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2.3-Repackage", origin_file_pattern="video_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2.3-Repackage", origin_file_pattern="audio_vae_decoder.safetensors", **vram_config), + ModelConfig(model_id="DiffSynth-Studio/LTX-2.3-Repackage", origin_file_pattern="audio_vocoder.safetensors", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="google/gemma-3-12b-it-qat-q4_0-unquantized"), +) +pipe.load_lora(pipe.dit, "models/train/LTX2.3-T2AV_lora/epoch-4.safetensors") +prompt = "A beautiful sunset over the ocean." +negative_prompt = "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." +height, width, num_frames = 512, 768, 121 +video, audio = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + seed=43, + height=height, + width=width, + num_frames=num_frames, + tiled=True, + cfg_scale=4.0 +) +write_video_audio_ltx2( + video=video, + audio=audio, + output_path='ltx2_onestage.mp4', + fps=24, + audio_sample_rate=pipe.audio_vocoder.output_sampling_rate, +) diff --git a/examples/minimax_h3/model_inference/MiniMax-H3-FL2VA-Turbo.py b/examples/minimax_h3/model_inference/MiniMax-H3-FL2VA-Turbo.py new file mode 100644 index 0000000000000000000000000000000000000000..b53b266e5abd69fcba9072bedcffbf18098a9337 --- /dev/null +++ b/examples/minimax_h3/model_inference/MiniMax-H3-FL2VA-Turbo.py @@ -0,0 +1,56 @@ +import torch +from diffsynth.pipelines.minimax_h3_audio_video import MiniMaxH3Pipeline, ModelConfig +from diffsynth.utils.data.audio_video import write_video_audio +from modelscope import dataset_snapshot_download +from PIL import Image + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cpu", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = MiniMaxH3Pipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="MiniMax/MiniMax-H3", origin_file_pattern="FL2VA/text_encoder/model*.safetensors", **vram_config), + ModelConfig(model_id="MiniMax/MiniMax-H3", origin_file_pattern="FL2VA/transformer/model*.safetensors", **vram_config), + ModelConfig(model_id="MiniMax/MiniMax-H3", origin_file_pattern="FL2VA/video_vae/source/model.safetensors", **vram_config), + ModelConfig(model_id="MiniMax/MiniMax-H3", origin_file_pattern="FL2VA/audio_vae/model.safetensors", **vram_config), + ], + processor_config=ModelConfig(model_id="MiniMax/MiniMax-H3", origin_file_pattern="FL2VA/processor/"), + vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 2, +) + +pipe.load_lora(pipe.dit, ModelConfig(model_id="lightx2v/Minimax-h3-Turbo", origin_file_pattern="minimax_h3_fl2v_turbo_4step_v1.0_768p_bf16.safetensors")) + +# Text -> Video + Audio +prompt = "A girl is very happy, she is speaking in english: “I enjoy working with Diffsynth-Studio, it's a perfect framework.”" +video, audio = pipe( + prompt=prompt, + height=480, width=832, num_frames=124, num_inference_steps=4, seed=0, flow_shift=6.0 +) +write_video_audio( + video=video, audio=audio, + output_path="t2va.mp4", fps=24, audio_sample_rate=32000, +) + +# Text + First Frame + Last Frame -> Video + Audio +dataset_snapshot_download(dataset_id="DiffSynth-Studio/diffsynth_example_dataset", local_dir="data/diffsynth_example_dataset", allow_file_pattern="minimax_h3/MiniMax-H3-FL2VA/*") +first_frame = Image.open("data/diffsynth_example_dataset/minimax_h3/MiniMax-H3-FL2VA/first.png") +last_frame = Image.open("data/diffsynth_example_dataset/minimax_h3/MiniMax-H3-FL2VA/last.png") +prompt = "室内家庭争吵短剧场景，竖屏短剧质感，真实真人表演，中式家庭/小饭馆室内环境，暖色灯光，背景有红色装饰和书法字幅，浅景深，情绪强烈，剪辑节奏紧凑。表演要求：真实短剧表演风格，不要夸张舞台腔。男人的语气是愤怒、委屈、急切的反驳，他说“你到底想干什么？”；中老年女性的语气是尖锐、强势、咄咄逼人的质问，她说“你必须赔钱！”。两人之间有强烈对峙感，节奏逐步升级。画面风格：竖屏9:16，手机短剧质感，真人实拍感，浅景深，室内暖光，中近景为主，频繁正反打剪辑，背景保持生活化，不要科幻、不要古装、不要动画感。画面中不要出现任何字幕、文字、平台水印或贴片。 " +video, audio = pipe( + prompt=prompt, + height=832, width=480, num_frames=124, num_inference_steps=4, seed=0, flow_shift=6.0, + keyframes=[first_frame, last_frame], keyframe_indices=[0, -1], +) +write_video_audio( + video=video, audio=audio, + output_path="fl2va.mp4", fps=24, audio_sample_rate=32000, +) diff --git a/examples/minimax_h3/model_inference/MiniMax-H3-FL2VA.py b/examples/minimax_h3/model_inference/MiniMax-H3-FL2VA.py new file mode 100644 index 0000000000000000000000000000000000000000..865c769c11cd01ac351d4d7a6d7a696ca842ed51 --- /dev/null +++ b/examples/minimax_h3/model_inference/MiniMax-H3-FL2VA.py @@ -0,0 +1,54 @@ +import torch +from diffsynth.pipelines.minimax_h3_audio_video import MiniMaxH3Pipeline, ModelConfig +from diffsynth.utils.data.audio_video import write_video_audio +from modelscope import dataset_snapshot_download +from PIL import Image + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cpu", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = MiniMaxH3Pipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="MiniMax/MiniMax-H3", origin_file_pattern="FL2VA/text_encoder/model*.safetensors", **vram_config), + ModelConfig(model_id="MiniMax/MiniMax-H3", origin_file_pattern="FL2VA/transformer/model*.safetensors", **vram_config), + ModelConfig(model_id="MiniMax/MiniMax-H3", origin_file_pattern="FL2VA/video_vae/source/model.safetensors", **vram_config), + ModelConfig(model_id="MiniMax/MiniMax-H3", origin_file_pattern="FL2VA/audio_vae/model.safetensors", **vram_config), + ], + processor_config=ModelConfig(model_id="MiniMax/MiniMax-H3", origin_file_pattern="FL2VA/processor/"), + vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 2, +) + +# Text -> Video + Audio +prompt = "A girl is very happy, she is speaking in english: “I enjoy working with Diffsynth-Studio, it's a perfect framework.”" +video, audio = pipe( + prompt=prompt, + height=480, width=832, num_frames=124, num_inference_steps=50, seed=0, +) +write_video_audio( + video=video, audio=audio, + output_path="t2va.mp4", fps=24, audio_sample_rate=32000, +) + +# Text + First Frame + Last Frame -> Video + Audio +dataset_snapshot_download(dataset_id="DiffSynth-Studio/diffsynth_example_dataset", local_dir="data/diffsynth_example_dataset", allow_file_pattern="minimax_h3/MiniMax-H3-FL2VA/*") +first_frame = Image.open("data/diffsynth_example_dataset/minimax_h3/MiniMax-H3-FL2VA/first.png") +last_frame = Image.open("data/diffsynth_example_dataset/minimax_h3/MiniMax-H3-FL2VA/last.png") +prompt = "室内家庭争吵短剧场景，竖屏短剧质感，真实真人表演，中式家庭/小饭馆室内环境，暖色灯光，背景有红色装饰和书法字幅，浅景深，情绪强烈，剪辑节奏紧凑。表演要求：真实短剧表演风格，不要夸张舞台腔。男人的语气是愤怒、委屈、急切的反驳，他说“你到底想干什么？”；中老年女性的语气是尖锐、强势、咄咄逼人的质问，她说“你必须赔钱！”。两人之间有强烈对峙感，节奏逐步升级。画面风格：竖屏9:16，手机短剧质感，真人实拍感，浅景深，室内暖光，中近景为主，频繁正反打剪辑，背景保持生活化，不要科幻、不要古装、不要动画感。画面中不要出现任何字幕、文字、平台水印或贴片。 " +video, audio = pipe( + prompt=prompt, + height=832, width=480, num_frames=124, num_inference_steps=50, seed=0, + keyframes=[first_frame, last_frame], keyframe_indices=[0, -1], +) +write_video_audio( + video=video, audio=audio, + output_path="fl2va.mp4", fps=24, audio_sample_rate=32000, +) diff --git a/examples/minimax_h3/model_inference/MiniMax-H3-FP8-Pruned-FL2VA.py b/examples/minimax_h3/model_inference/MiniMax-H3-FP8-Pruned-FL2VA.py new file mode 100644 index 0000000000000000000000000000000000000000..fb15db0bc2a923010f5b103b5a27be7fc5df8502 --- /dev/null +++ b/examples/minimax_h3/model_inference/MiniMax-H3-FP8-Pruned-FL2VA.py @@ -0,0 +1,52 @@ +import torch +from diffsynth.pipelines.minimax_h3_audio_video import MiniMaxH3Pipeline, ModelConfig +from diffsynth.utils.data.audio_video import write_video_audio +from modelscope import dataset_snapshot_download +from PIL import Image + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cpu", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = MiniMaxH3Pipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="Comfy-Org/MiniMax-H3", origin_file_pattern="diffusion_models/minimax_h3_fl2va_pruned_fp8_scaled.safetensors", **vram_config), + ModelConfig(model_id="MiniMax/MiniMax-H3", origin_file_pattern="FL2VA/text_encoder/model*.safetensors", **vram_config), + ModelConfig(model_id="MiniMax/MiniMax-H3", origin_file_pattern="FL2VA/video_vae/source/model.safetensors", **vram_config), + ModelConfig(model_id="MiniMax/MiniMax-H3", origin_file_pattern="FL2VA/audio_vae/model.safetensors", **vram_config), + ], + processor_config=ModelConfig(model_id="MiniMax/MiniMax-H3", origin_file_pattern="FL2VA/processor/"), + vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 2, +) + +prompt = "A girl is very happy, she is speaking in english: “I enjoy working with Diffsynth-Studio, it’s a perfect framework.”" +video, audio = pipe( + prompt=prompt, + height=480, width=832, num_frames=124, num_inference_steps=50, seed=0, +) +write_video_audio( + video=video, audio=audio, + output_path="t2va.mp4", fps=24, audio_sample_rate=32000, +) + +dataset_snapshot_download(dataset_id="DiffSynth-Studio/diffsynth_example_dataset", local_dir="data/diffsynth_example_dataset", allow_file_pattern="minimax_h3/MiniMax-H3-Pruned-FL2VA/*") +first_frame = Image.open("data/diffsynth_example_dataset/minimax_h3/MiniMax-H3-Pruned-FL2VA/first.png") +last_frame = Image.open("data/diffsynth_example_dataset/minimax_h3/MiniMax-H3-Pruned-FL2VA/last.png") +prompt = "室内家庭争吵短剧场景，竖屏短剧质感，真实真人表演，中式家庭/小饭馆室内环境，暖色灯光，背景有红色装饰和书法字幅，浅景深，情绪强烈，剪辑节奏紧凑。表演要求：真实短剧表演风格，不要夸张舞台腔。男人的语气是愤怒、委屈、急切的反驳，他说“你到底想干什么？”；中老年女性的语气是尖锐、强势、咆咆逼人的质问，她说“你必须赔钱！”。两人之间有强烈对峨感，节奏逐步升级。画面风格：竖屏9:16，手机短剧质感，真人实拍感，浅景深，室内暖光，中近景为主，频繁正反打剪辑，背景保持生活化，不要科幻、不要古装、不要动画感。画面中不要出现任何字幕、文字、平台水印或贴片。 " +video, audio = pipe( + prompt=prompt, + height=832, width=480, num_frames=124, num_inference_steps=50, seed=0, + keyframes=[first_frame, last_frame], keyframe_indices=[0, -1], +) +write_video_audio( + video=video, audio=audio, + output_path="fl2va.mp4", fps=24, audio_sample_rate=32000, +) diff --git a/examples/minimax_h3/model_inference/MiniMax-H3-FP8-Pruned-Ref2VA.py b/examples/minimax_h3/model_inference/MiniMax-H3-FP8-Pruned-Ref2VA.py new file mode 100644 index 0000000000000000000000000000000000000000..83879c56a0f2747e16529cfc9b1d9dfa729dd452 --- /dev/null +++ b/examples/minimax_h3/model_inference/MiniMax-H3-FP8-Pruned-Ref2VA.py @@ -0,0 +1,69 @@ +import torch +from PIL import Image +from diffsynth.pipelines.minimax_h3_audio_video import MiniMaxH3Pipeline, ModelConfig +from diffsynth.utils.data.audio_video import write_video_audio, read_video_audio +from diffsynth.utils.data.audio import read_audio +from modelscope import dataset_snapshot_download + +def align_frame_count(frame_count): + current = max(int(frame_count), 1) + while current % 17 != 5: + current += 1 + return current + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cpu", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} +pipe = MiniMaxH3Pipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="Comfy-Org/MiniMax-H3", origin_file_pattern="diffusion_models/minimax_h3_ref2va_pruned_fp8_scaled.safetensors", **vram_config), + ModelConfig(model_id="MiniMax/MiniMax-H3", origin_file_pattern="Ref2VA/text_encoder/model*.safetensors", **vram_config), + ModelConfig(model_id="MiniMax/MiniMax-H3", origin_file_pattern="Ref2VA/video_vae/source/model.safetensors", **vram_config), + ModelConfig(model_id="MiniMax/MiniMax-H3", origin_file_pattern="Ref2VA/audio_vae/model.safetensors", **vram_config), + ], + processor_config=ModelConfig(model_id="MiniMax/MiniMax-H3", origin_file_pattern="Ref2VA/processor/"), + vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 2, +) + +dataset_snapshot_download(dataset_id="DiffSynth-Studio/diffsynth_example_dataset", local_dir="data/diffsynth_example_dataset", allow_file_pattern="minimax_h3/MiniMax-H3-Pruned-Ref2VA/*") +ref_image = Image.open("data/diffsynth_example_dataset/minimax_h3/MiniMax-H3-Pruned-Ref2VA/0.png").convert("RGB") +prompt = "一个网站页面，网站页面UI设计，网站动效，视频展示了流畅的网页向下滚动效果。一个极具爆发力与动感的产品官网风格产品落地页 UI/UX 演示视频，核心展示主体是该产品图片1。页面采用粗犷有力、倾斜的超大号无衬线字体进行张扬的排版。背景有极具速度感的动态光影、暗色碳纤维或运动透气网眼纹理在交织变换。视频展示了节奏紧凑、充满力量感的网页向下滚动效果，以及鼠标悬停时强烈的视觉放大与颜色反转等 UI 交互动作。" +video, audio = pipe( + prompt=prompt, + height=480, width=832, num_frames=124, num_inference_steps=50, seed=42, + references=[{"type": "image", "image": Image.open("data/diffsynth_example_dataset/minimax_h3/MiniMax-H3-Pruned-Ref2VA/0.png").convert("RGB")}] +) +write_video_audio( + video=video, audio=audio, + output_path="ti2va.mp4", fps=24, audio_sample_rate=32000, +) + +ref_video, ref_video_audio, ref_video_sample_rate = read_video_audio( + "data/diffsynth_example_dataset/minimax_h3/MiniMax-H3-Pruned-Ref2VA/video.mp4", + height=480, width=832, num_frames=124, fps=24, audio_sample_rate=pipe.audio_vae.sample_rate, +) +ref_voice, ref_voice_sample_rate = read_audio( + "data/diffsynth_example_dataset/minimax_h3/MiniMax-H3-Pruned-Ref2VA/voice.mp3", duration=len(ref_video) / 24, resample=True, resample_rate=pipe.audio_vae.sample_rate, +) +prompt = "subject_definitions:\n is the young man with short wavy blonde hair, wearing a bright pink suit jacket, matching pink trousers, an unbuttoned white shirt, and silver rings, holding a small black lamb in his arms in