Buckets:

hf-doc-build
/

doc

Files

xet

hf-doc-build/doc / diffusers /v0.43.0 /zh /using-diffusers /text-img2vid.md

mishig

2 days ago

preview code

download

raw

18.3 kB

# 视频生成

视频生成模型将图像生成（可视为单帧视频）扩展至同时处理与空间和时间相关的数据。确保所有这些文本、空间和时间数据在帧与帧之间保持一致且对齐，是生成长时长、高分辨率视频的一大挑战。

现代视频模型采用扩散 Transformer（DiT）架构来应对这一挑战。该架构降低了计算成本，并能够更高效地扩展至更大规模、更高质量的图像和视频数据。

请查看下方这些视频模型的部分能力展示。

```py
# pip install ftfy
import torch
import numpy as np
from diffusers import AutoModel, WanPipeline
from diffusers.hooks.group_offloading import apply_group_offloading
from diffusers.utils import export_to_video, load_image
from transformers import UMT5EncoderModel
text_encoder = UMT5EncoderModel.from_pretrained("Wan-AI/Wan2.1-T2V-14B-Diffusers", subfolder="text_encoder", dtype=torch.bfloat16)
vae = AutoModel.from_pretrained("Wan-AI/Wan2.1-T2V-14B-Diffusers", subfolder="vae", dtype=torch.float32)
transformer = AutoModel.from_pretrained("Wan-AI/Wan2.1-T2V-14B-Diffusers", subfolder="transformer", dtype=torch.bfloat16)
# group-offloading
onload_device = torch.device("cuda")
offload_device = torch.device("cpu")
apply_group_offloading(text_encoder,
    onload_device=onload_device,
    offload_device=offload_device,
    offload_type="block_level",
    num_blocks_per_group=4
)
transformer.enable_group_offload(
    onload_device=onload_device,
    offload_device=offload_device,
    offload_type="leaf_level",
    use_stream=True
)
pipeline = WanPipeline.from_pretrained(
    "Wan-AI/Wan2.1-T2V-14B-Diffusers",
    vae=vae,
    transformer=transformer,
    text_encoder=text_encoder,
    dtype=torch.bfloat16
)
pipeline.to("cuda")  # or "mps", "xpu", "cpu"
prompt = """
The camera rushes from far to near in a low-angle shot, 
revealing a white ferret on a log. It plays, leaps into the water, and emerges, as the camera zooms in 
for a close-up. Water splashes berry bushes nearby, while moss, snow, and leaves blanket the ground. 
Birch trees and a light blue sky frame the scene, with ferns in the foreground. Side lighting casts dynamic 
shadows and warm highlights. Medium composition, front view, low angle, with depth of field.
"""
negative_prompt = """
Bright tones, overexposed, static, blurred details, subtitles, style, works, paintings, images, static, overall gray, worst quality, 
low quality, JPEG compression residue, ugly, incomplete, extra fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured, 
misshapen limbs, fused fingers, still picture, messy background, three legs, many people in the background, walking backwards
"""
output = pipeline(
    prompt=prompt,
    negative_prompt=negative_prompt,
    num_frames=81,
    guidance_scale=5.0,
).frames[0]
export_to_video(output, "output.mp4", fps=16)
```

```py
import torch
from diffusers importAutoModel, HunyuanVideoPipeline
from diffusers.quantizers import PipelineQuantizationConfig
from diffusers.utils import export_to_video
# quantize weights to int4 with bitsandbytes
pipeline_quant_config = PipelineQuantizationConfig(
  quant_backend="bitsandbytes_4bit",
  quant_kwargs={
    "load_in_4bit": True,
    "bnb_4bit_quant_type": "nf4",
    "bnb_4bit_compute_dtype": torch.bfloat16
    },
  components_to_quantize="transformer"
)
pipeline = HunyuanVideoPipeline.from_pretrained(
    "hunyuanvideo-community/HunyuanVideo",
    quantization_config=pipeline_quant_config,
    dtype=torch.bfloat16,
)
# model-offloading and tiling
pipeline.enable_model_cpu_offload()
pipeline.vae.enable_tiling()
prompt = "A fluffy teddy bear sits on a bed of soft pillows surrounded by children's toys."
video = pipeline(prompt=prompt, num_frames=61, num_inference_steps=30).frames[0]
export_to_video(video, "output.mp4", fps=15)
```

```py
import torch
from diffusers import LTXPipeline, AutoModel
from diffusers.hooks import apply_group_offloading
from diffusers.utils import export_to_video
# fp8 layerwise weight-casting
transformer = AutoModel.from_pretrained(
    "Lightricks/LTX-Video",
    subfolder="transformer",
    dtype=torch.bfloat16
)
transformer.enable_layerwise_casting(
    storage_dtype=torch.float8_e4m3fn, compute_dtype=torch.bfloat16
)
pipeline = LTXPipeline.from_pretrained("Lightricks/LTX-Video", transformer=transformer, dtype=torch.bfloat16)
# group-offloading
onload_device = torch.device("cuda")
offload_device = torch.device("cpu")
pipeline.transformer.enable_group_offload(onload_device=onload_device, offload_device=offload_device, offload_type="leaf_level", use_stream=True)
apply_group_offloading(pipeline.text_encoder, onload_device=onload_device, offload_type="block_level", num_blocks_per_group=2)
apply_group_offloading(pipeline.vae, onload_device=onload_device, offload_type="leaf_level")
prompt = """
A woman with long brown hair and light skin smiles at another woman with long blonde hair. The woman with brown hair wears a black jacket and has a small, barely noticeable mole on her right cheek. The camera angle is a close-up, focused on the woman with brown hair's face. The lighting is warm and natural, likely from the setting sun, casting a soft glow on the scene. The scene appears to be real-life footage
"""
negative_prompt = "worst quality, inconsistent motion, blurry, jittery, distorted"
video = pipeline(
    prompt=prompt,
    negative_prompt=negative_prompt,
    width=768,
    height=512,
    num_frames=161,
    decode_timestep=0.03,
    decode_noise_scale=0.025,
    num_inference_steps=50,
).frames[0]
export_to_video(video, "output.mp4", fps=24)
```

本指南将介绍视频生成的基础知识，例如应配置哪些参数以及如何降低其内存占用。

> [!TIP]
> 如果您想进一步了解如何使用特定模型，请参阅其 pipeline API 模型卡。

## Pipeline 参数

在 pipeline 中，有多个参数会影响视频生成的质量或速度。尝试不同的参数值对于找到合适的质量与速度之间的平衡至关重要。

### num_frames
帧是作为其他帧序列中的一部分进行播放的静态图像，用于产生运动效果或视频。使用 `num_frames` 控制每秒生成的帧数。增加 `num_frames` 可以提升感知到的运动流畅度和视觉连贯性，对于包含动态内容的视频尤为重要。较高的 `num_frames` 值还会增加视频时长。

某些视频模型在推理时需要更具体的 `num_frames` 值。例如，[HunyuanVideoPipeline](/docs/diffusers/v0.43.0/zh/api/pipelines/hunyuan_video#diffusers.HunyuanVideoPipeline) 建议使用 `(4 * num_frames) +1` 来计算 `num_frames`。请务必查阅 pipeline 的 API 模型卡，以确认是否有推荐的值。

```py
import torch
from diffusers import LTXPipeline
from diffusers.utils import export_to_video
pipeline = LTXPipeline.from_pretrained(
    "Lightricks/LTX-Video", dtype=torch.bfloat16
).to("cuda")  # or "mps", "xpu", "cpu"
prompt = """
A woman with long brown hair and light skin smiles at another woman with long blonde hair. The woman 
with brown hair wears a black jacket and has a small, barely noticeable mole on her right cheek. The 
camera angle is a close-up, focused on the woman with brown hair's face. The lighting is warm and 
natural, likely from the setting sun, casting a soft glow on the scene. The scene appears to be 
real-life footage
"""
video = pipeline(
    prompt=prompt,
    negative_prompt=negative_prompt,
    width=768,
    height=512,
    num_frames=161,
    decode_timestep=0.03,
    decode_noise_scale=0.025,
    num_inference_steps=50,
).frames[0]
export_to_video(video, "output.mp4", fps=24)
```

### guidance_scale
引导尺度（guidance scale）或 "cfg" 控制生成的帧在多大程度上遵循输入条件（文本、图像或两者兼有）。增大 `guidance_scale` 会生成与输入条件更相似、细节更丰富的帧，但可能会引入伪影并降低输出的多样性。较低的 `guidance_scale` 值会鼓励更宽松的提示词遵循，从而增加输出的多样性，但细节可能不够出色。如果该值过低，模型可能会完全忽略提示词，生成随机噪声。
```py
import torch
from diffusers import CogVideoXPipeline, CogVideoXTransformer3DModel
from diffusers.utils import export_to_video
pipeline = CogVideoXPipeline.from_pretrained(
  "THUDM/CogVideoX-2b",
  dtype=torch.float16
).to("cuda")  # or "mps", "xpu", "cpu"

prompt = """
A detailed wooden toy ship with intricately carved masts and sails is seen gliding smoothly over
a plush, blue carpet that mimics the waves of the sea. The ship's hull is painted a rich brown, 
with tiny windows. The carpet, soft and textured, provides a perfect backdrop, resembling an 
oceanic expanse. Surrounding the ship are various other toys and children's items, hinting at 
a playful environment. The scene captures the innocence and imagination of childhood, 
with the toy ship's journey symbolizing endless adventures in a whimsical, indoor setting.
"""

video = pipeline(
  prompt=prompt,
  guidance_scale=6,
  num_inference_steps=50
).frames[0]
export_to_video(video, "output.mp4", fps=8)
```
### negative_prompt

负向提示词（negative prompt）有助于排除你不想在生成视频中看到的内容。它通常用于通过引导模型远离“模糊、扭曲、丑陋”等不良元素，来提升生成视频的质量和一致性。这可以生成更干净、更聚焦的视频。

```py
# pip install ftfy
import torch
from diffusers import WanPipeline
from diffusers.schedulers.scheduling_unipc_multistep import UniPCMultistepScheduler
from diffusers.utils import export_to_video
vae = AutoencoderKLWan.from_pretrained(
  "Wan-AI/Wan2.1-T2V-14B-Diffusers", subfolder="vae", dtype=torch.float32
)
pipeline = WanPipeline.from_pretrained(
  "Wan-AI/Wan2.1-T2V-14B-Diffusers", vae=vae, dtype=torch.bfloat16
)
pipeline.scheduler = UniPCMultistepScheduler.from_config(
  pipeline.scheduler.config, flow_shift=5.0
)
pipeline.to("cuda")  # or "mps", "xpu", "cpu"
pipeline.load_lora_weights("benjamin-paine/steamboat-willie-14b", adapter_name="steamboat-willie")
pipeline.set_adapters("steamboat-willie")
pipeline.enable_model_cpu_offload()
# use "steamboat willie style" to trigger the LoRA
prompt = """
steamboat willie style, golden era animation, The camera rushes from far to near in a low-angle shot, 
revealing a white ferret on a log. It plays, leaps into the water, and emerges, as the camera zooms in 
for a close-up. Water splashes berry bushes nearby, while moss, snow, and leaves blanket the ground. 
Birch trees and a light blue sky frame the scene, with ferns in the foreground. Side lighting casts 
dynamic shadows and warm highlights. Medium composition, front view, low angle, with depth of field.
"""
output = pipeline(
  prompt=prompt,
  num_frames=81,
  guidance_scale=5.0,
).frames[0]
export_to_video(output, "output.mp4", fps=16)
```

## 减少内存使用

像 [HunyuanVideoPipeline](/docs/diffusers/v0.43.0/zh/api/pipelines/hunyuan_video#diffusers.HunyuanVideoPipeline) 和 [WanPipeline](/docs/diffusers/v0.43.0/zh/api/pipelines/wan#diffusers.WanPipeline) 这样拥有 100 亿以上参数的最新视频模型需要大量内存，通常会超出消费级硬件的可用内存。Diffusers 提供了多种技术来降低这些大型模型的内存需求。

> [!TIP]
> 有关其他节省内存技术的详细信息，请参阅 [减少内存使用](../optimization/memory) 指南。

其中一种技术是[group-offloading](../optimization/memory#group-offloading)，它会在模型内部层组（例如`torch.nn.Sequential`）未被使用时，将其卸载到 CPU。这些层仅在计算需要时才会被加载，从而避免在 GPU 上存储**所有**模型组件。对于像[WanPipeline](/docs/diffusers/v0.43.0/zh/api/pipelines/wan#diffusers.WanPipeline)这样的 14B 参数模型，group-offloading 可将所需显存降低至约 13GB。

```py
# pip install ftfy
import torch
import numpy as np
from diffusers import AutoModel, WanPipeline
from diffusers.hooks.group_offloading import apply_group_offloading
from diffusers.utils import export_to_video, load_image
from transformers import UMT5EncoderModel
text_encoder = UMT5EncoderModel.from_pretrained("Wan-AI/Wan2.1-T2V-14B-Diffusers", subfolder="text_encoder", dtype=torch.bfloat16)
vae = AutoModel.from_pretrained("Wan-AI/Wan2.1-T2V-14B-Diffusers", subfolder="vae", dtype=torch.float32)
transformer = AutoModel.from_pretrained("Wan-AI/Wan2.1-T2V-14B-Diffusers", subfolder="transformer", dtype=torch.bfloat16)
# group-offloading
onload_device = torch.device("cuda")
offload_device = torch.device("cpu")
apply_group_offloading(text_encoder,
    onload_device=onload_device,
    offload_device=offload_device,
    offload_type="block_level",
    num_blocks_per_group=4
)
transformer.enable_group_offload(
    onload_device=onload_device,
    offload_device=offload_device,
    offload_type="leaf_level",
    use_stream=True
)
pipeline = WanPipeline.from_pretrained(
    "Wan-AI/Wan2.1-T2V-14B-Diffusers",
    vae=vae,
    transformer=transformer,
    text_encoder=text_encoder,
    dtype=torch.bfloat16
)
pipeline.to("cuda")  # or "mps", "xpu", "cpu"
prompt = """
The camera rushes from far to near in a low-angle shot, 
revealing a white ferret on a log. It plays, leaps into the water, and emerges, as the camera zooms in 
for a close-up. Water splashes berry bushes nearby, while moss, snow, and leaves blanket the ground. 
Birch trees and a light blue sky frame the scene, with ferns in the foreground. Side lighting casts dynamic 
shadows and warm highlights. Medium composition, front view, low angle, with depth of field.
"""
negative_prompt = """
Bright tones, overexposed, static, blurred details, subtitles, style, works, paintings, images, static, overall gray, worst quality, 
low quality, JPEG compression residue, ugly, incomplete, extra fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured, 
misshapen limbs, fused fingers, still picture, messy background, three legs, many people in the background, walking backwards
"""
output = pipeline(
    prompt=prompt,
    negative_prompt=negative_prompt,
    num_frames=81,
    guidance_scale=5.0,
).frames[0]
export_to_video(output, "output.mp4", fps=16)
```

另一种减少内存占用的方法是考虑对模型进行量化，即将模型权重存储为更低精度的数据类型。不过，量化可能会影响视频质量，具体取决于所使用的视频模型。有关不同支持的量化后端的详细信息，请参阅量化[概述](../quantization/overview)。

以下示例使用 [bitsandbytes](../quantization/bitsandbytes) 对模型进行量化。

```py
# pip install ftfy
import torch
from diffusers import WanPipeline
from diffusers import AutoModel, WanPipeline
from diffusers.quantizers import PipelineQuantizationConfig
from diffusers.schedulers.scheduling_unipc_multistep import UniPCMultistepScheduler
from transformers import UMT5EncoderModel
from diffusers.utils import export_to_video
# quantize transformer and text encoder weights with bitsandbytes
pipeline_quant_config = PipelineQuantizationConfig(
  quant_backend="bitsandbytes_4bit",
  quant_kwargs={"load_in_4bit": True},
  components_to_quantize=["transformer", "text_encoder"]
)
vae = AutoModel.from_pretrained(
  "Wan-AI/Wan2.1-T2V-14B-Diffusers", subfolder="vae", dtype=torch.float32
)
pipeline = WanPipeline.from_pretrained(
  "Wan-AI/Wan2.1-T2V-14B-Diffusers", vae=vae, quantization_config=pipeline_quant_config, dtype=torch.bfloat16
)
pipeline.scheduler = UniPCMultistepScheduler.from_config(
  pipeline.scheduler.config, flow_shift=5.0
)
pipeline.to("cuda")  # or "mps", "xpu", "cpu"
pipeline.load_lora_weights("benjamin-paine/steamboat-willie-14b", adapter_name="steamboat-willie")
pipeline.set_adapters("steamboat-willie")
pipeline.enable_model_cpu_offload()
# use "steamboat willie style" to trigger the LoRA
prompt = """
steamboat willie style, golden era animation, The camera rushes from far to near in a low-angle shot, 
revealing a white ferret on a log. It plays, leaps into the water, and emerges, as the camera zooms in 
for a close-up. Water splashes berry bushes nearby, while moss, snow, and leaves blanket the ground. 
Birch trees and a light blue sky frame the scene, with ferns in the foreground. Side lighting casts 
dynamic shadows and warm highlights. Medium composition, front view, low angle, with depth of field.
"""
output = pipeline(
  prompt=prompt,
  num_frames=81,
  guidance_scale=5.0,
).frames[0]
export_to_video(output, "output.mp4", fps=16)
```

## 推理速度

[torch.compile](https://docs.pytorch.org/tutorials/intermediate/torch_compile_tutorial.html) 可以通过使用优化后的内核来加速推理。首次编译耗时较长，但编译完成后速度会显著提升。最佳实践是只编译一次 pipeline，然后在不更改任何设置的情况下多次使用该 pipeline。任何更改（例如图像尺寸的变化）都会触发重新编译。

以下示例会编译流水线中的 transformer，并使用 `"max-autotune"` 模式以最大化性能。

```py
import torch
from diffusers import CogVideoXPipeline, CogVideoXTransformer3DModel
from diffusers.utils import export_to_video
pipeline = CogVideoXPipeline.from_pretrained(
  "THUDM/CogVideoX-2b",
  dtype=torch.float16
).to("cuda")  # or "mps", "xpu", "cpu"
# torch.compile
pipeline.transformer.to(memory_format=torch.channels_last)
pipeline.transformer = torch.compile(
    pipeline.transformer, mode="max-autotune", fullgraph=True
)
prompt = """
A detailed wooden toy ship with intricately carved masts and sails is seen gliding smoothly over a plush, blue carpet that mimics the waves of the sea. 
The ship's hull is painted a rich brown, with tiny windows. The carpet, soft and textured, provides a perfect backdrop, resembling an oceanic expanse. 
Surrounding the ship are various other toys and children's items, hinting at a playful environment. The scene captures the innocence and imagination of childhood, 
with the toy ship's journey symbolizing endless adventures in a whimsical, indoor setting.
"""
video = pipeline(
  prompt=prompt,
  guidance_scale=6,
  num_inference_steps=50
).frames[0]
export_to_video(video, "output.mp4", fps=8)
```

Xet Storage Details

Size:: 18.3 kB
Xet hash:: 310a1c0b2f1c6359f667b190d278b948c57366b0232ae3ad542263bf9a460bfc

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.

	# 视频生成

	视频生成模型将图像生成（可视为单帧视频）扩展至同时处理与空间和时间相关的数据。确保所有这些文本、空间和时间数据在帧与帧之间保持一致且对齐，是生成长时长、高分辨率视频的一大挑战。

	现代视频模型采用扩散 Transformer（DiT）架构来应对这一挑战。该架构降低了计算成本，并能够更高效地扩展至更大规模、更高质量的图像和视频数据。

	请查看下方这些视频模型的部分能力展示。

	```py
	# pip install ftfy
	import torch
	import numpy as np
	from diffusers import AutoModel, WanPipeline
	from diffusers.hooks.group_offloading import apply_group_offloading
	from diffusers.utils import export_to_video, load_image
	from transformers import UMT5EncoderModel

	text_encoder = UMT5EncoderModel.from_pretrained("Wan-AI/Wan2.1-T2V-14B-Diffusers", subfolder="text_encoder", dtype=torch.bfloat16)
	vae = AutoModel.from_pretrained("Wan-AI/Wan2.1-T2V-14B-Diffusers", subfolder="vae", dtype=torch.float32)
	transformer = AutoModel.from_pretrained("Wan-AI/Wan2.1-T2V-14B-Diffusers", subfolder="transformer", dtype=torch.bfloat16)

	# group-offloading
	onload_device = torch.device("cuda")
	offload_device = torch.device("cpu")
	apply_group_offloading(text_encoder,
	onload_device=onload_device,
	offload_device=offload_device,
	offload_type="block_level",
	num_blocks_per_group=4
	)
	transformer.enable_group_offload(
	onload_device=onload_device,
	offload_device=offload_device,
	offload_type="leaf_level",
	use_stream=True
	)

	pipeline = WanPipeline.from_pretrained(
	"Wan-AI/Wan2.1-T2V-14B-Diffusers",
	vae=vae,
	transformer=transformer,
	text_encoder=text_encoder,
	dtype=torch.bfloat16
	)
	pipeline.to("cuda") # or "mps", "xpu", "cpu"

	prompt = """
	The camera rushes from far to near in a low-angle shot,
	revealing a white ferret on a log. It plays, leaps into the water, and emerges, as the camera zooms in
	for a close-up. Water splashes berry bushes nearby, while moss, snow, and leaves blanket the ground.
	Birch trees and a light blue sky frame the scene, with ferns in the foreground. Side lighting casts dynamic
	shadows and warm highlights. Medium composition, front view, low angle, with depth of field.
	"""
	negative_prompt = """
	Bright tones, overexposed, static, blurred details, subtitles, style, works, paintings, images, static, overall gray, worst quality,
	low quality, JPEG compression residue, ugly, incomplete, extra fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured,
	misshapen limbs, fused fingers, still picture, messy background, three legs, many people in the background, walking backwards
	"""

	output = pipeline(
	prompt=prompt,
	negative_prompt=negative_prompt,
	num_frames=81,
	guidance_scale=5.0,
	).frames[0]
	export_to_video(output, "output.mp4", fps=16)
	```

	```py
	import torch
	from diffusers importAutoModel, HunyuanVideoPipeline
	from diffusers.quantizers import PipelineQuantizationConfig
	from diffusers.utils import export_to_video

	# quantize weights to int4 with bitsandbytes
	pipeline_quant_config = PipelineQuantizationConfig(
	quant_backend="bitsandbytes_4bit",
	quant_kwargs={
	"load_in_4bit": True,
	"bnb_4bit_quant_type": "nf4",
	"bnb_4bit_compute_dtype": torch.bfloat16
	},
	components_to_quantize="transformer"
	)

	pipeline = HunyuanVideoPipeline.from_pretrained(
	"hunyuanvideo-community/HunyuanVideo",
	quantization_config=pipeline_quant_config,
	dtype=torch.bfloat16,
	)

	# model-offloading and tiling
	pipeline.enable_model_cpu_offload()
	pipeline.vae.enable_tiling()

	prompt = "A fluffy teddy bear sits on a bed of soft pillows surrounded by children's toys."
	video = pipeline(prompt=prompt, num_frames=61, num_inference_steps=30).frames[0]
	export_to_video(video, "output.mp4", fps=15)
	```

	```py
	import torch
	from diffusers import LTXPipeline, AutoModel
	from diffusers.hooks import apply_group_offloading
	from diffusers.utils import export_to_video

	# fp8 layerwise weight-casting
	transformer = AutoModel.from_pretrained(
	"Lightricks/LTX-Video",
	subfolder="transformer",
	dtype=torch.bfloat16
	)
	transformer.enable_layerwise_casting(
	storage_dtype=torch.float8_e4m3fn, compute_dtype=torch.bfloat16
	)

	pipeline = LTXPipeline.from_pretrained("Lightricks/LTX-Video", transformer=transformer, dtype=torch.bfloat16)

	# group-offloading
	onload_device = torch.device("cuda")
	offload_device = torch.device("cpu")
	pipeline.transformer.enable_group_offload(onload_device=onload_device, offload_device=offload_device, offload_type="leaf_level", use_stream=True)
	apply_group_offloading(pipeline.text_encoder, onload_device=onload_device, offload_type="block_level", num_blocks_per_group=2)
	apply_group_offloading(pipeline.vae, onload_device=onload_device, offload_type="leaf_level")

	prompt = """
	A woman with long brown hair and light skin smiles at another woman with long blonde hair. The woman with brown hair wears a black jacket and has a small, barely noticeable mole on her right cheek. The camera angle is a close-up, focused on the woman with brown hair's face. The lighting is warm and natural, likely from the setting sun, casting a soft glow on the scene. The scene appears to be real-life footage
	"""
	negative_prompt = "worst quality, inconsistent motion, blurry, jittery, distorted"

	video = pipeline(
	prompt=prompt,
	negative_prompt=negative_prompt,
	width=768,
	height=512,
	num_frames=161,
	decode_timestep=0.03,
	decode_noise_scale=0.025,
	num_inference_steps=50,
	).frames[0]
	export_to_video(video, "output.mp4", fps=24)
	```

	本指南将介绍视频生成的基础知识，例如应配置哪些参数以及如何降低其内存占用。

	> [!TIP]
	> 如果您想进一步了解如何使用特定模型，请参阅其 pipeline API 模型卡。

	## Pipeline 参数

	在 pipeline 中，有多个参数会影响视频生成的质量或速度。尝试不同的参数值对于找到合适的质量与速度之间的平衡至关重要。

	### num_frames

	帧是作为其他帧序列中的一部分进行播放的静态图像，用于产生运动效果或视频。使用 `num_frames` 控制每秒生成的帧数。增加 `num_frames` 可以提升感知到的运动流畅度和视觉连贯性，对于包含动态内容的视频尤为重要。较高的 `num_frames` 值还会增加视频时长。

	某些视频模型在推理时需要更具体的 `num_frames` 值。例如，[HunyuanVideoPipeline](/docs/diffusers/v0.43.0/zh/api/pipelines/hunyuan_video#diffusers.HunyuanVideoPipeline) 建议使用 `(4 * num_frames) +1` 来计算 `num_frames`。请务必查阅 pipeline 的 API 模型卡，以确认是否有推荐的值。

	```py
	import torch
	from diffusers import LTXPipeline
	from diffusers.utils import export_to_video

	pipeline = LTXPipeline.from_pretrained(
	"Lightricks/LTX-Video", dtype=torch.bfloat16
	).to("cuda") # or "mps", "xpu", "cpu"

	prompt = """
	A woman with long brown hair and light skin smiles at another woman with long blonde hair. The woman
	with brown hair wears a black jacket and has a small, barely noticeable mole on her right cheek. The
	camera angle is a close-up, focused on the woman with brown hair's face. The lighting is warm and
	natural, likely from the setting sun, casting a soft glow on the scene. The scene appears to be
	real-life footage
	"""

	video = pipeline(
	prompt=prompt,
	negative_prompt=negative_prompt,
	width=768,
	height=512,
	num_frames=161,
	decode_timestep=0.03,
	decode_noise_scale=0.025,
	num_inference_steps=50,
	).frames[0]
	export_to_video(video, "output.mp4", fps=24)
	```

	### guidance_scale

	引导尺度（guidance scale）或 "cfg" 控制生成的帧在多大程度上遵循输入条件（文本、图像或两者兼有）。增大 `guidance_scale` 会生成与输入条件更相似、细节更丰富的帧，但可能会引入伪影并降低输出的多样性。较低的 `guidance_scale` 值会鼓励更宽松的提示词遵循，从而增加输出的多样性，但细节可能不够出色。如果该值过低，模型可能会完全忽略提示词，生成随机噪声。

	```py
	import torch
	from diffusers import CogVideoXPipeline, CogVideoXTransformer3DModel
	from diffusers.utils import export_to_video

	pipeline = CogVideoXPipeline.from_pretrained(
	"THUDM/CogVideoX-2b",
	dtype=torch.float16
	).to("cuda") # or "mps", "xpu", "cpu"

	prompt = """
	A detailed wooden toy ship with intricately carved masts and sails is seen gliding smoothly over
	a plush, blue carpet that mimics the waves of the sea. The ship's hull is painted a rich brown,
	with tiny windows. The carpet, soft and textured, provides a perfect backdrop, resembling an
	oceanic expanse. Surrounding the ship are various other toys and children's items, hinting at
	a playful environment. The scene captures the innocence and imagination of childhood,
	with the toy ship's journey symbolizing endless adventures in a whimsical, indoor setting.
	"""

	video = pipeline(
	prompt=prompt,
	guidance_scale=6,
	num_inference_steps=50
	).frames[0]
	export_to_video(video, "output.mp4", fps=8)
	```

	### negative_prompt

	负向提示词（negative prompt）有助于排除你不想在生成视频中看到的内容。它通常用于通过引导模型远离“模糊、扭曲、丑陋”等不良元素，来提升生成视频的质量和一致性。这可以生成更干净、更聚焦的视频。

	```py
	# pip install ftfy
	import torch
	from diffusers import WanPipeline
	from diffusers.schedulers.scheduling_unipc_multistep import UniPCMultistepScheduler
	from diffusers.utils import export_to_video

	vae = AutoencoderKLWan.from_pretrained(
	"Wan-AI/Wan2.1-T2V-14B-Diffusers", subfolder="vae", dtype=torch.float32
	)
	pipeline = WanPipeline.from_pretrained(
	"Wan-AI/Wan2.1-T2V-14B-Diffusers", vae=vae, dtype=torch.bfloat16
	)
	pipeline.scheduler = UniPCMultistepScheduler.from_config(
	pipeline.scheduler.config, flow_shift=5.0
	)
	pipeline.to("cuda") # or "mps", "xpu", "cpu"

	pipeline.load_lora_weights("benjamin-paine/steamboat-willie-14b", adapter_name="steamboat-willie")
	pipeline.set_adapters("steamboat-willie")

	pipeline.enable_model_cpu_offload()

	# use "steamboat willie style" to trigger the LoRA
	prompt = """
	steamboat willie style, golden era animation, The camera rushes from far to near in a low-angle shot,
	revealing a white ferret on a log. It plays, leaps into the water, and emerges, as the camera zooms in
	for a close-up. Water splashes berry bushes nearby, while moss, snow, and leaves blanket the ground.
	Birch trees and a light blue sky frame the scene, with ferns in the foreground. Side lighting casts
	dynamic shadows and warm highlights. Medium composition, front view, low angle, with depth of field.
	"""

	output = pipeline(
	prompt=prompt,
	num_frames=81,
	guidance_scale=5.0,
	).frames[0]
	export_to_video(output, "output.mp4", fps=16)
	```

	## 减少内存使用

	像 [HunyuanVideoPipeline](/docs/diffusers/v0.43.0/zh/api/pipelines/hunyuan_video#diffusers.HunyuanVideoPipeline) 和 [WanPipeline](/docs/diffusers/v0.43.0/zh/api/pipelines/wan#diffusers.WanPipeline) 这样拥有 100 亿以上参数的最新视频模型需要大量内存，通常会超出消费级硬件的可用内存。Diffusers 提供了多种技术来降低这些大型模型的内存需求。

	> [!TIP]
	> 有关其他节省内存技术的详细信息，请参阅 [减少内存使用](../optimization/memory) 指南。

	其中一种技术是[group-offloading](../optimization/memory#group-offloading)，它会在模型内部层组（例如`torch.nn.Sequential`）未被使用时，将其卸载到 CPU。这些层仅在计算需要时才会被加载，从而避免在 GPU 上存储所有模型组件。对于像[WanPipeline](/docs/diffusers/v0.43.0/zh/api/pipelines/wan#diffusers.WanPipeline)这样的 14B 参数模型，group-offloading 可将所需显存降低至约 13GB。

	```py
	# pip install ftfy
	import torch
	import numpy as np
	from diffusers import AutoModel, WanPipeline
	from diffusers.hooks.group_offloading import apply_group_offloading
	from diffusers.utils import export_to_video, load_image
	from transformers import UMT5EncoderModel

	text_encoder = UMT5EncoderModel.from_pretrained("Wan-AI/Wan2.1-T2V-14B-Diffusers", subfolder="text_encoder", dtype=torch.bfloat16)
	vae = AutoModel.from_pretrained("Wan-AI/Wan2.1-T2V-14B-Diffusers", subfolder="vae", dtype=torch.float32)
	transformer = AutoModel.from_pretrained("Wan-AI/Wan2.1-T2V-14B-Diffusers", subfolder="transformer", dtype=torch.bfloat16)

	# group-offloading
	onload_device = torch.device("cuda")
	offload_device = torch.device("cpu")
	apply_group_offloading(text_encoder,
	onload_device=onload_device,
	offload_device=offload_device,
	offload_type="block_level",
	num_blocks_per_group=4
	)
	transformer.enable_group_offload(
	onload_device=onload_device,
	offload_device=offload_device,
	offload_type="leaf_level",
	use_stream=True
	)

	pipeline = WanPipeline.from_pretrained(
	"Wan-AI/Wan2.1-T2V-14B-Diffusers",
	vae=vae,
	transformer=transformer,
	text_encoder=text_encoder,
	dtype=torch.bfloat16
	)
	pipeline.to("cuda") # or "mps", "xpu", "cpu"

	prompt = """
	The camera rushes from far to near in a low-angle shot,
	revealing a white ferret on a log. It plays, leaps into the water, and emerges, as the camera zooms in
	for a close-up. Water splashes berry bushes nearby, while moss, snow, and leaves blanket the ground.
	Birch trees and a light blue sky frame the scene, with ferns in the foreground. Side lighting casts dynamic
	shadows and warm highlights. Medium composition, front view, low angle, with depth of field.
	"""
	negative_prompt = """
	Bright tones, overexposed, static, blurred details, subtitles, style, works, paintings, images, static, overall gray, worst quality,
	low quality, JPEG compression residue, ugly, incomplete, extra fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured,
	misshapen limbs, fused fingers, still picture, messy background, three legs, many people in the background, walking backwards
	"""

	output = pipeline(
	prompt=prompt,
	negative_prompt=negative_prompt,
	num_frames=81,
	guidance_scale=5.0,
	).frames[0]
	export_to_video(output, "output.mp4", fps=16)
	```

	另一种减少内存占用的方法是考虑对模型进行量化，即将模型权重存储为更低精度的数据类型。不过，量化可能会影响视频质量，具体取决于所使用的视频模型。有关不同支持的量化后端的详细信息，请参阅量化[概述](../quantization/overview)。

	以下示例使用 [bitsandbytes](../quantization/bitsandbytes) 对模型进行量化。

	```py
	# pip install ftfy

	import torch
	from diffusers import WanPipeline
	from diffusers import AutoModel, WanPipeline
	from diffusers.quantizers import PipelineQuantizationConfig
	from diffusers.schedulers.scheduling_unipc_multistep import UniPCMultistepScheduler
	from transformers import UMT5EncoderModel
	from diffusers.utils import export_to_video

	# quantize transformer and text encoder weights with bitsandbytes
	pipeline_quant_config = PipelineQuantizationConfig(
	quant_backend="bitsandbytes_4bit",
	quant_kwargs={"load_in_4bit": True},
	components_to_quantize=["transformer", "text_encoder"]
	)

	vae = AutoModel.from_pretrained(
	"Wan-AI/Wan2.1-T2V-14B-Diffusers", subfolder="vae", dtype=torch.float32
	)
	pipeline = WanPipeline.from_pretrained(
	"Wan-AI/Wan2.1-T2V-14B-Diffusers", vae=vae, quantization_config=pipeline_quant_config, dtype=torch.bfloat16
	)
	pipeline.scheduler = UniPCMultistepScheduler.from_config(
	pipeline.scheduler.config, flow_shift=5.0
	)
	pipeline.to("cuda") # or "mps", "xpu", "cpu"

	pipeline.load_lora_weights("benjamin-paine/steamboat-willie-14b", adapter_name="steamboat-willie")
	pipeline.set_adapters("steamboat-willie")

	pipeline.enable_model_cpu_offload()

	# use "steamboat willie style" to trigger the LoRA
	prompt = """
	steamboat willie style, golden era animation, The camera rushes from far to near in a low-angle shot,
	revealing a white ferret on a log. It plays, leaps into the water, and emerges, as the camera zooms in
	for a close-up. Water splashes berry bushes nearby, while moss, snow, and leaves blanket the ground.
	Birch trees and a light blue sky frame the scene, with ferns in the foreground. Side lighting casts
	dynamic shadows and warm highlights. Medium composition, front view, low angle, with depth of field.
	"""

	output = pipeline(
	prompt=prompt,
	num_frames=81,
	guidance_scale=5.0,
	).frames[0]
	export_to_video(output, "output.mp4", fps=16)
	```

	## 推理速度

	[torch.compile](https://docs.pytorch.org/tutorials/intermediate/torch_compile_tutorial.html) 可以通过使用优化后的内核来加速推理。首次编译耗时较长，但编译完成后速度会显著提升。最佳实践是只编译一次 pipeline，然后在不更改任何设置的情况下多次使用该 pipeline。任何更改（例如图像尺寸的变化）都会触发重新编译。

	以下示例会编译流水线中的 transformer，并使用 `"max-autotune"` 模式以最大化性能。

	```py
	import torch
	from diffusers import CogVideoXPipeline, CogVideoXTransformer3DModel
	from diffusers.utils import export_to_video

	pipeline = CogVideoXPipeline.from_pretrained(
	"THUDM/CogVideoX-2b",
	dtype=torch.float16
	).to("cuda") # or "mps", "xpu", "cpu"

	# torch.compile
	pipeline.transformer.to(memory_format=torch.channels_last)
	pipeline.transformer = torch.compile(
	pipeline.transformer, mode="max-autotune", fullgraph=True
	)

	prompt = """
	A detailed wooden toy ship with intricately carved masts and sails is seen gliding smoothly over a plush, blue carpet that mimics the waves of the sea.
	The ship's hull is painted a rich brown, with tiny windows. The carpet, soft and textured, provides a perfect backdrop, resembling an oceanic expanse.
	Surrounding the ship are various other toys and children's items, hinting at a playful environment. The scene captures the innocence and imagination of childhood,
	with the toy ship's journey symbolizing endless adventures in a whimsical, indoor setting.
	"""

	video = pipeline(
	prompt=prompt,
	guidance_scale=6,
	num_inference_steps=50
	).frames[0]
	export_to_video(video, "output.mp4", fps=8)
	```