Buckets:

hf-doc-build
/

doc

Files

xet

hf-doc-build/doc / diffusers /v0.43.0 /pt /using-diffusers /text-img2vid.md

mishig

2 days ago

preview code

download

raw

19.2 kB

# Geração de vídeo

Modelos de geração de vídeo estendem a geração de imagens (que pode ser considerada um vídeo de 1 quadro) para também processar dados relacionados a espaço e tempo. Garantir que todos esses dados — texto, espaço, tempo — permaneçam consistentes e alinhados de um quadro para o outro é um grande desafio na geração de vídeos longos e de alta resolução.

Modelos modernos de vídeo abordam esse desafio com a arquitetura de difusão transformer (DiT). Isso reduz os custos computacionais e permite uma escalabilidade mais eficiente para dados de imagem e vídeo maiores e de maior qualidade.

Confira abaixo o que alguns desses modelos de vídeo são capazes de fazer.

```py
# pip install ftfy
import torch
import numpy as np
from diffusers import AutoModel, WanPipeline
from diffusers.hooks.group_offloading import apply_group_offloading
from diffusers.utils import export_to_video, load_image
from transformers import UMT5EncoderModel
text_encoder = UMT5EncoderModel.from_pretrained("Wan-AI/Wan2.1-T2V-14B-Diffusers", subfolder="text_encoder", dtype=torch.bfloat16)
vae = AutoModel.from_pretrained("Wan-AI/Wan2.1-T2V-14B-Diffusers", subfolder="vae", dtype=torch.float32)
transformer = AutoModel.from_pretrained("Wan-AI/Wan2.1-T2V-14B-Diffusers", subfolder="transformer", dtype=torch.bfloat16)
# group-offloading
onload_device = torch.device("cuda")
offload_device = torch.device("cpu")
apply_group_offloading(text_encoder,
    onload_device=onload_device,
    offload_device=offload_device,
    offload_type="block_level",
    num_blocks_per_group=4
)
transformer.enable_group_offload(
    onload_device=onload_device,
    offload_device=offload_device,
    offload_type="leaf_level",
    use_stream=True
)
pipeline = WanPipeline.from_pretrained(
    "Wan-AI/Wan2.1-T2V-14B-Diffusers",
    vae=vae,
    transformer=transformer,
    text_encoder=text_encoder,
    dtype=torch.bfloat16
)
pipeline.to("cuda")  # or "mps", "xpu", "cpu"
prompt = """
The camera rushes from far to near in a low-angle shot, 
revealing a white ferret on a log. It plays, leaps into the water, and emerges, as the camera zooms in 
for a close-up. Water splashes berry bushes nearby, while moss, snow, and leaves blanket the ground. 
Birch trees and a light blue sky frame the scene, with ferns in the foreground. Side lighting casts dynamic 
shadows and warm highlights. Medium composition, front view, low angle, with depth of field.
"""
negative_prompt = """
Bright tones, overexposed, static, blurred details, subtitles, style, works, paintings, images, static, overall gray, worst quality, 
low quality, JPEG compression residue, ugly, incomplete, extra fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured, 
misshapen limbs, fused fingers, still picture, messy background, three legs, many people in the background, walking backwards
"""
output = pipeline(
    prompt=prompt,
    negative_prompt=negative_prompt,
    num_frames=81,
    guidance_scale=5.0,
).frames[0]
export_to_video(output, "output.mp4", fps=16)
```

```py
import torch
from diffusers importAutoModel, HunyuanVideoPipeline
from diffusers.quantizers import PipelineQuantizationConfig
from diffusers.utils import export_to_video
# quantize weights to int4 with bitsandbytes
pipeline_quant_config = PipelineQuantizationConfig(
  quant_backend="bitsandbytes_4bit",
  quant_kwargs={
    "load_in_4bit": True,
    "bnb_4bit_quant_type": "nf4",
    "bnb_4bit_compute_dtype": torch.bfloat16
    },
  components_to_quantize="transformer"
)
pipeline = HunyuanVideoPipeline.from_pretrained(
    "hunyuanvideo-community/HunyuanVideo",
    quantization_config=pipeline_quant_config,
    dtype=torch.bfloat16,
)
# model-offloading and tiling
pipeline.enable_model_cpu_offload()
pipeline.vae.enable_tiling()
prompt = "A fluffy teddy bear sits on a bed of soft pillows surrounded by children's toys."
video = pipeline(prompt=prompt, num_frames=61, num_inference_steps=30).frames[0]
export_to_video(video, "output.mp4", fps=15)
```

```py
import torch
from diffusers import LTXPipeline, AutoModel
from diffusers.hooks import apply_group_offloading
from diffusers.utils import export_to_video
# fp8 layerwise weight-casting
transformer = AutoModel.from_pretrained(
    "Lightricks/LTX-Video",
    subfolder="transformer",
    dtype=torch.bfloat16
)
transformer.enable_layerwise_casting(
    storage_dtype=torch.float8_e4m3fn, compute_dtype=torch.bfloat16
)
pipeline = LTXPipeline.from_pretrained("Lightricks/LTX-Video", transformer=transformer, dtype=torch.bfloat16)
# group-offloading
onload_device = torch.device("cuda")
offload_device = torch.device("cpu")
pipeline.transformer.enable_group_offload(onload_device=onload_device, offload_device=offload_device, offload_type="leaf_level", use_stream=True)
apply_group_offloading(pipeline.text_encoder, onload_device=onload_device, offload_type="block_level", num_blocks_per_group=2)
apply_group_offloading(pipeline.vae, onload_device=onload_device, offload_type="leaf_level")
prompt = """
A woman with long brown hair and light skin smiles at another woman with long blonde hair. The woman with brown hair wears a black jacket and has a small, barely noticeable mole on her right cheek. The camera angle is a close-up, focused on the woman with brown hair's face. The lighting is warm and natural, likely from the setting sun, casting a soft glow on the scene. The scene appears to be real-life footage
"""
negative_prompt = "worst quality, inconsistent motion, blurry, jittery, distorted"
video = pipeline(
    prompt=prompt,
    negative_prompt=negative_prompt,
    width=768,
    height=512,
    num_frames=161,
    decode_timestep=0.03,
    decode_noise_scale=0.025,
    num_inference_steps=50,
).frames[0]
export_to_video(video, "output.mp4", fps=24)
```

Este guia abordará os fundamentos da geração de vídeo, como quais parâmetros configurar e como reduzir o uso de memória.

> [!TIP]
> Se você tiver interesse em aprender mais sobre como usar um modelo específico, consulte o cartão do modelo da API de pipeline deles.

## Parâmetros do Pipeline

Existem vários parâmetros para configurar no pipeline que afetarão a qualidade ou a velocidade da geração de vídeo. Experimentar com diferentes valores de parâmetros é importante para descobrir o equilíbrio adequado entre qualidade e velocidade.

### num_frames
Um frame é uma imagem estática que é reproduzida em sequência com outros frames para criar movimento ou um vídeo. Controle o número de frames gerados por segundo com `num_frames`. Aumentar `num_frames` aumenta a suavidade percebida do movimento e a coerência visual, o que é especialmente importante para vídeos com conteúdo dinâmico. Um valor maior de `num_frames` também aumenta a duração do vídeo.

Alguns modelos de vídeo exigem valores mais específicos de `num_frames` para inferência. Por exemplo, o [HunyuanVideoPipeline](/docs/diffusers/v0.43.0/pt/api/pipelines/hunyuan_video#diffusers.HunyuanVideoPipeline) recomenda calcular o `num_frames` com `(4 * num_frames) +1`. Verifique sempre o cartão de modelo da API de um pipeline para ver se há um valor recomendado.

```py
import torch
from diffusers import LTXPipeline
from diffusers.utils import export_to_video
pipeline = LTXPipeline.from_pretrained(
    "Lightricks/LTX-Video", dtype=torch.bfloat16
).to("cuda")  # or "mps", "xpu", "cpu"
prompt = """
A woman with long brown hair and light skin smiles at another woman with long blonde hair. The woman 
with brown hair wears a black jacket and has a small, barely noticeable mole on her right cheek. The 
camera angle is a close-up, focused on the woman with brown hair's face. The lighting is warm and 
natural, likely from the setting sun, casting a soft glow on the scene. The scene appears to be 
real-life footage
"""
video = pipeline(
    prompt=prompt,
    negative_prompt=negative_prompt,
    width=768,
    height=512,
    num_frames=161,
    decode_timestep=0.03,
    decode_noise_scale=0.025,
    num_inference_steps=50,
).frames[0]
export_to_video(video, "output.mp4", fps=24)
```

### guidance_scale
A escala de orientação ou "cfg" controla o quão de perto os frames gerados seguem a condição de entrada (texto, imagem ou ambos). Aumentar `guidance_scale` gera frames que se assemelham mais de perto às condições de entrada e inclui detalhes mais finos, mas corre o risco de introduzir artefatos e reduzir a diversidade da saída. Valores mais baixos de `guidance_scale` encorajam uma aderência mais frouxa ao prompt e uma maior variedade de saída, mas os detalhes podem não ser tão bons. Se for muito baixo, pode ignorar completamente seu prompt e gerar ruído aleatório.
```py
import torch
from diffusers import CogVideoXPipeline, CogVideoXTransformer3DModel
from diffusers.utils import export_to_video
pipeline = CogVideoXPipeline.from_pretrained(
  "THUDM/CogVideoX-2b",
  dtype=torch.float16
).to("cuda")  # or "mps", "xpu", "cpu"

prompt = """
A detailed wooden toy ship with intricately carved masts and sails is seen gliding smoothly over
a plush, blue carpet that mimics the waves of the sea. The ship's hull is painted a rich brown, 
with tiny windows. The carpet, soft and textured, provides a perfect backdrop, resembling an 
oceanic expanse. Surrounding the ship are various other toys and children's items, hinting at 
a playful environment. The scene captures the innocence and imagination of childhood, 
with the toy ship's journey symbolizing endless adventures in a whimsical, indoor setting.
"""

video = pipeline(
  prompt=prompt,
  guidance_scale=6,
  num_inference_steps=50
).frames[0]
export_to_video(video, "output.mp4", fps=8)
```
### negative_prompt

Um prompt negativo é útil para excluir elementos que você não deseja ver no vídeo gerado. Ele é comumente usado para refinar a qualidade e o alinhamento do vídeo gerado, afastando o modelo de elementos indesejados, como "blurry, distorted, ugly". Isso pode resultar em vídeos mais limpos e focados.

```py
# pip install ftfy
import torch
from diffusers import WanPipeline
from diffusers.schedulers.scheduling_unipc_multistep import UniPCMultistepScheduler
from diffusers.utils import export_to_video
vae = AutoencoderKLWan.from_pretrained(
  "Wan-AI/Wan2.1-T2V-14B-Diffusers", subfolder="vae", dtype=torch.float32
)
pipeline = WanPipeline.from_pretrained(
  "Wan-AI/Wan2.1-T2V-14B-Diffusers", vae=vae, dtype=torch.bfloat16
)
pipeline.scheduler = UniPCMultistepScheduler.from_config(
  pipeline.scheduler.config, flow_shift=5.0
)
pipeline.to("cuda")  # or "mps", "xpu", "cpu"
pipeline.load_lora_weights("benjamin-paine/steamboat-willie-14b", adapter_name="steamboat-willie")
pipeline.set_adapters("steamboat-willie")
pipeline.enable_model_cpu_offload()
# use "steamboat willie style" to trigger the LoRA
prompt = """
steamboat willie style, golden era animation, The camera rushes from far to near in a low-angle shot, 
revealing a white ferret on a log. It plays, leaps into the water, and emerges, as the camera zooms in 
for a close-up. Water splashes berry bushes nearby, while moss, snow, and leaves blanket the ground. 
Birch trees and a light blue sky frame the scene, with ferns in the foreground. Side lighting casts 
dynamic shadows and warm highlights. Medium composition, front view, low angle, with depth of field.
"""
output = pipeline(
  prompt=prompt,
  num_frames=81,
  guidance_scale=5.0,
).frames[0]
export_to_video(output, "output.mp4", fps=16)
```

## Reduzir o uso de memória

Modelos de vídeo recentes, como [HunyuanVideoPipeline](/docs/diffusers/v0.43.0/pt/api/pipelines/hunyuan_video#diffusers.HunyuanVideoPipeline) e [WanPipeline](/docs/diffusers/v0.43.0/pt/api/pipelines/wan#diffusers.WanPipeline), que possuem mais de 10 bilhões de parâmetros, exigem uma grande quantidade de memória, frequentemente excedendo a memória disponível em hardware de consumo. O Diffusers oferece várias técnicas para reduzir os requisitos de memória desses modelos grandes.

> [!TIP]
> Consulte o guia [Reduzir o uso de memória](../optimization/memory) para mais detalhes sobre outras técnicas de economia de memória.

Uma dessas técnicas é o [group-offloading](../optimization/memory#group-offloading), que transfere grupos de camadas internas do modelo (como `torch.nn.Sequential`) para a CPU quando não estão em uso. Essas camadas são carregadas apenas quando são necessárias para o cálculo, evitando armazenar **todos** os componentes do modelo na GPU. Para um modelo com 14 bilhões de parâmetros, como o [WanPipeline](/docs/diffusers/v0.43.0/pt/api/pipelines/wan#diffusers.WanPipeline), o group-offloading pode reduzir a memória necessária para cerca de 13 GB de VRAM.

```py
# pip install ftfy
import torch
import numpy as np
from diffusers import AutoModel, WanPipeline
from diffusers.hooks.group_offloading import apply_group_offloading
from diffusers.utils import export_to_video, load_image
from transformers import UMT5EncoderModel
text_encoder = UMT5EncoderModel.from_pretrained("Wan-AI/Wan2.1-T2V-14B-Diffusers", subfolder="text_encoder", dtype=torch.bfloat16)
vae = AutoModel.from_pretrained("Wan-AI/Wan2.1-T2V-14B-Diffusers", subfolder="vae", dtype=torch.float32)
transformer = AutoModel.from_pretrained("Wan-AI/Wan2.1-T2V-14B-Diffusers", subfolder="transformer", dtype=torch.bfloat16)
# group-offloading
onload_device = torch.device("cuda")
offload_device = torch.device("cpu")
apply_group_offloading(text_encoder,
    onload_device=onload_device,
    offload_device=offload_device,
    offload_type="block_level",
    num_blocks_per_group=4
)
transformer.enable_group_offload(
    onload_device=onload_device,
    offload_device=offload_device,
    offload_type="leaf_level",
    use_stream=True
)
pipeline = WanPipeline.from_pretrained(
    "Wan-AI/Wan2.1-T2V-14B-Diffusers",
    vae=vae,
    transformer=transformer,
    text_encoder=text_encoder,
    dtype=torch.bfloat16
)
pipeline.to("cuda")  # or "mps", "xpu", "cpu"
prompt = """
The camera rushes from far to near in a low-angle shot, 
revealing a white ferret on a log. It plays, leaps into the water, and emerges, as the camera zooms in 
for a close-up. Water splashes berry bushes nearby, while moss, snow, and leaves blanket the ground. 
Birch trees and a light blue sky frame the scene, with ferns in the foreground. Side lighting casts dynamic 
shadows and warm highlights. Medium composition, front view, low angle, with depth of field.
"""
negative_prompt = """
Bright tones, overexposed, static, blurred details, subtitles, style, works, paintings, images, static, overall gray, worst quality, 
low quality, JPEG compression residue, ugly, incomplete, extra fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured, 
misshapen limbs, fused fingers, still picture, messy background, three legs, many people in the background, walking backwards
"""
output = pipeline(
    prompt=prompt,
    negative_prompt=negative_prompt,
    num_frames=81,
    guidance_scale=5.0,
).frames[0]
export_to_video(output, "output.mp4", fps=16)
```

Outra opção para reduzir a memória é considerar a quantização de um modelo, que armazena os pesos do modelo em um tipo de dados de menor precisão. No entanto, a quantização pode impactar a qualidade do vídeo, dependendo do modelo de vídeo específico. Consulte a [Visão geral](../quantization/overview) da quantização para saber mais sobre os diferentes backends de quantização suportados.

O exemplo abaixo usa [bitsandbytes](../quantization/bitsandbytes) para quantizar um modelo.

```py
# pip install ftfy
import torch
from diffusers import WanPipeline
from diffusers import AutoModel, WanPipeline
from diffusers.quantizers import PipelineQuantizationConfig
from diffusers.schedulers.scheduling_unipc_multistep import UniPCMultistepScheduler
from transformers import UMT5EncoderModel
from diffusers.utils import export_to_video
# quantize transformer and text encoder weights with bitsandbytes
pipeline_quant_config = PipelineQuantizationConfig(
  quant_backend="bitsandbytes_4bit",
  quant_kwargs={"load_in_4bit": True},
  components_to_quantize=["transformer", "text_encoder"]
)
vae = AutoModel.from_pretrained(
  "Wan-AI/Wan2.1-T2V-14B-Diffusers", subfolder="vae", dtype=torch.float32
)
pipeline = WanPipeline.from_pretrained(
  "Wan-AI/Wan2.1-T2V-14B-Diffusers", vae=vae, quantization_config=pipeline_quant_config, dtype=torch.bfloat16
)
pipeline.scheduler = UniPCMultistepScheduler.from_config(
  pipeline.scheduler.config, flow_shift=5.0
)
pipeline.to("cuda")  # or "mps", "xpu", "cpu"
pipeline.load_lora_weights("benjamin-paine/steamboat-willie-14b", adapter_name="steamboat-willie")
pipeline.set_adapters("steamboat-willie")
pipeline.enable_model_cpu_offload()
# use "steamboat willie style" to trigger the LoRA
prompt = """
steamboat willie style, golden era animation, The camera rushes from far to near in a low-angle shot, 
revealing a white ferret on a log. It plays, leaps into the water, and emerges, as the camera zooms in 
for a close-up. Water splashes berry bushes nearby, while moss, snow, and leaves blanket the ground. 
Birch trees and a light blue sky frame the scene, with ferns in the foreground. Side lighting casts 
dynamic shadows and warm highlights. Medium composition, front view, low angle, with depth of field.
"""
output = pipeline(
  prompt=prompt,
  num_frames=81,
  guidance_scale=5.0,
).frames[0]
export_to_video(output, "output.mp4", fps=16)
```

## Velocidade de inferência

[torch.compile](https://docs.pytorch.org/tutorials/intermediate/torch_compile_tutorial.html) pode acelerar a inferência ao usar kernels otimizados. A compilação leva mais tempo na primeira vez, mas, uma vez compilado, é muito mais rápido. O ideal é compilar o pipeline uma vez e, em seguida, usá-lo várias vezes sem alterar nada. Uma alteração, como no tamanho da imagem, dispara a recompilação.

O exemplo abaixo compila o transformer no pipeline e usa o modo `"max-autotune"` para maximizar o desempenho.

```py
import torch
from diffusers import CogVideoXPipeline, CogVideoXTransformer3DModel
from diffusers.utils import export_to_video
pipeline = CogVideoXPipeline.from_pretrained(
  "THUDM/CogVideoX-2b",
  dtype=torch.float16
).to("cuda")  # or "mps", "xpu", "cpu"
# torch.compile
pipeline.transformer.to(memory_format=torch.channels_last)
pipeline.transformer = torch.compile(
    pipeline.transformer, mode="max-autotune", fullgraph=True
)
prompt = """
A detailed wooden toy ship with intricately carved masts and sails is seen gliding smoothly over a plush, blue carpet that mimics the waves of the sea. 
The ship's hull is painted a rich brown, with tiny windows. The carpet, soft and textured, provides a perfect backdrop, resembling an oceanic expanse. 
Surrounding the ship are various other toys and children's items, hinting at a playful environment. The scene captures the innocence and imagination of childhood, 
with the toy ship's journey symbolizing endless adventures in a whimsical, indoor setting.
"""
video = pipeline(
  prompt=prompt,
  guidance_scale=6,
  num_inference_steps=50
).frames[0]
export_to_video(video, "output.mp4", fps=8)
```

Xet Storage Details

Size:: 19.2 kB
Xet hash:: abd73bac0a67c1b12045152191ff07a03de05c140dfdb8194b70c2c31fd32804

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.

	# Geração de vídeo

	Modelos de geração de vídeo estendem a geração de imagens (que pode ser considerada um vídeo de 1 quadro) para também processar dados relacionados a espaço e tempo. Garantir que todos esses dados — texto, espaço, tempo — permaneçam consistentes e alinhados de um quadro para o outro é um grande desafio na geração de vídeos longos e de alta resolução.

	Modelos modernos de vídeo abordam esse desafio com a arquitetura de difusão transformer (DiT). Isso reduz os custos computacionais e permite uma escalabilidade mais eficiente para dados de imagem e vídeo maiores e de maior qualidade.

	Confira abaixo o que alguns desses modelos de vídeo são capazes de fazer.

	```py
	# pip install ftfy
	import torch
	import numpy as np
	from diffusers import AutoModel, WanPipeline
	from diffusers.hooks.group_offloading import apply_group_offloading
	from diffusers.utils import export_to_video, load_image
	from transformers import UMT5EncoderModel

	text_encoder = UMT5EncoderModel.from_pretrained("Wan-AI/Wan2.1-T2V-14B-Diffusers", subfolder="text_encoder", dtype=torch.bfloat16)
	vae = AutoModel.from_pretrained("Wan-AI/Wan2.1-T2V-14B-Diffusers", subfolder="vae", dtype=torch.float32)
	transformer = AutoModel.from_pretrained("Wan-AI/Wan2.1-T2V-14B-Diffusers", subfolder="transformer", dtype=torch.bfloat16)

	# group-offloading
	onload_device = torch.device("cuda")
	offload_device = torch.device("cpu")
	apply_group_offloading(text_encoder,
	onload_device=onload_device,
	offload_device=offload_device,
	offload_type="block_level",
	num_blocks_per_group=4
	)
	transformer.enable_group_offload(
	onload_device=onload_device,
	offload_device=offload_device,
	offload_type="leaf_level",
	use_stream=True
	)

	pipeline = WanPipeline.from_pretrained(
	"Wan-AI/Wan2.1-T2V-14B-Diffusers",
	vae=vae,
	transformer=transformer,
	text_encoder=text_encoder,
	dtype=torch.bfloat16
	)
	pipeline.to("cuda") # or "mps", "xpu", "cpu"

	prompt = """
	The camera rushes from far to near in a low-angle shot,
	revealing a white ferret on a log. It plays, leaps into the water, and emerges, as the camera zooms in
	for a close-up. Water splashes berry bushes nearby, while moss, snow, and leaves blanket the ground.
	Birch trees and a light blue sky frame the scene, with ferns in the foreground. Side lighting casts dynamic
	shadows and warm highlights. Medium composition, front view, low angle, with depth of field.
	"""
	negative_prompt = """
	Bright tones, overexposed, static, blurred details, subtitles, style, works, paintings, images, static, overall gray, worst quality,
	low quality, JPEG compression residue, ugly, incomplete, extra fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured,
	misshapen limbs, fused fingers, still picture, messy background, three legs, many people in the background, walking backwards
	"""

	output = pipeline(
	prompt=prompt,
	negative_prompt=negative_prompt,
	num_frames=81,
	guidance_scale=5.0,
	).frames[0]
	export_to_video(output, "output.mp4", fps=16)
	```

	```py
	import torch
	from diffusers importAutoModel, HunyuanVideoPipeline
	from diffusers.quantizers import PipelineQuantizationConfig
	from diffusers.utils import export_to_video

	# quantize weights to int4 with bitsandbytes
	pipeline_quant_config = PipelineQuantizationConfig(
	quant_backend="bitsandbytes_4bit",
	quant_kwargs={
	"load_in_4bit": True,
	"bnb_4bit_quant_type": "nf4",
	"bnb_4bit_compute_dtype": torch.bfloat16
	},
	components_to_quantize="transformer"
	)

	pipeline = HunyuanVideoPipeline.from_pretrained(
	"hunyuanvideo-community/HunyuanVideo",
	quantization_config=pipeline_quant_config,
	dtype=torch.bfloat16,
	)

	# model-offloading and tiling
	pipeline.enable_model_cpu_offload()
	pipeline.vae.enable_tiling()

	prompt = "A fluffy teddy bear sits on a bed of soft pillows surrounded by children's toys."
	video = pipeline(prompt=prompt, num_frames=61, num_inference_steps=30).frames[0]
	export_to_video(video, "output.mp4", fps=15)
	```

	```py
	import torch
	from diffusers import LTXPipeline, AutoModel
	from diffusers.hooks import apply_group_offloading
	from diffusers.utils import export_to_video

	# fp8 layerwise weight-casting
	transformer = AutoModel.from_pretrained(
	"Lightricks/LTX-Video",
	subfolder="transformer",
	dtype=torch.bfloat16
	)
	transformer.enable_layerwise_casting(
	storage_dtype=torch.float8_e4m3fn, compute_dtype=torch.bfloat16
	)

	pipeline = LTXPipeline.from_pretrained("Lightricks/LTX-Video", transformer=transformer, dtype=torch.bfloat16)

	# group-offloading
	onload_device = torch.device("cuda")
	offload_device = torch.device("cpu")
	pipeline.transformer.enable_group_offload(onload_device=onload_device, offload_device=offload_device, offload_type="leaf_level", use_stream=True)
	apply_group_offloading(pipeline.text_encoder, onload_device=onload_device, offload_type="block_level", num_blocks_per_group=2)
	apply_group_offloading(pipeline.vae, onload_device=onload_device, offload_type="leaf_level")

	prompt = """
	A woman with long brown hair and light skin smiles at another woman with long blonde hair. The woman with brown hair wears a black jacket and has a small, barely noticeable mole on her right cheek. The camera angle is a close-up, focused on the woman with brown hair's face. The lighting is warm and natural, likely from the setting sun, casting a soft glow on the scene. The scene appears to be real-life footage
	"""
	negative_prompt = "worst quality, inconsistent motion, blurry, jittery, distorted"

	video = pipeline(
	prompt=prompt,
	negative_prompt=negative_prompt,
	width=768,
	height=512,
	num_frames=161,
	decode_timestep=0.03,
	decode_noise_scale=0.025,
	num_inference_steps=50,
	).frames[0]
	export_to_video(video, "output.mp4", fps=24)
	```

	Este guia abordará os fundamentos da geração de vídeo, como quais parâmetros configurar e como reduzir o uso de memória.

	> [!TIP]
	> Se você tiver interesse em aprender mais sobre como usar um modelo específico, consulte o cartão do modelo da API de pipeline deles.

	## Parâmetros do Pipeline

	Existem vários parâmetros para configurar no pipeline que afetarão a qualidade ou a velocidade da geração de vídeo. Experimentar com diferentes valores de parâmetros é importante para descobrir o equilíbrio adequado entre qualidade e velocidade.

	### num_frames

	Um frame é uma imagem estática que é reproduzida em sequência com outros frames para criar movimento ou um vídeo. Controle o número de frames gerados por segundo com `num_frames`. Aumentar `num_frames` aumenta a suavidade percebida do movimento e a coerência visual, o que é especialmente importante para vídeos com conteúdo dinâmico. Um valor maior de `num_frames` também aumenta a duração do vídeo.

	Alguns modelos de vídeo exigem valores mais específicos de `num_frames` para inferência. Por exemplo, o [HunyuanVideoPipeline](/docs/diffusers/v0.43.0/pt/api/pipelines/hunyuan_video#diffusers.HunyuanVideoPipeline) recomenda calcular o `num_frames` com `(4 * num_frames) +1`. Verifique sempre o cartão de modelo da API de um pipeline para ver se há um valor recomendado.

	```py
	import torch
	from diffusers import LTXPipeline
	from diffusers.utils import export_to_video

	pipeline = LTXPipeline.from_pretrained(
	"Lightricks/LTX-Video", dtype=torch.bfloat16
	).to("cuda") # or "mps", "xpu", "cpu"

	prompt = """
	A woman with long brown hair and light skin smiles at another woman with long blonde hair. The woman
	with brown hair wears a black jacket and has a small, barely noticeable mole on her right cheek. The
	camera angle is a close-up, focused on the woman with brown hair's face. The lighting is warm and
	natural, likely from the setting sun, casting a soft glow on the scene. The scene appears to be
	real-life footage
	"""

	video = pipeline(
	prompt=prompt,
	negative_prompt=negative_prompt,
	width=768,
	height=512,
	num_frames=161,
	decode_timestep=0.03,
	decode_noise_scale=0.025,
	num_inference_steps=50,
	).frames[0]
	export_to_video(video, "output.mp4", fps=24)
	```

	### guidance_scale

	A escala de orientação ou "cfg" controla o quão de perto os frames gerados seguem a condição de entrada (texto, imagem ou ambos). Aumentar `guidance_scale` gera frames que se assemelham mais de perto às condições de entrada e inclui detalhes mais finos, mas corre o risco de introduzir artefatos e reduzir a diversidade da saída. Valores mais baixos de `guidance_scale` encorajam uma aderência mais frouxa ao prompt e uma maior variedade de saída, mas os detalhes podem não ser tão bons. Se for muito baixo, pode ignorar completamente seu prompt e gerar ruído aleatório.

	```py
	import torch
	from diffusers import CogVideoXPipeline, CogVideoXTransformer3DModel
	from diffusers.utils import export_to_video

	pipeline = CogVideoXPipeline.from_pretrained(
	"THUDM/CogVideoX-2b",
	dtype=torch.float16
	).to("cuda") # or "mps", "xpu", "cpu"

	prompt = """
	A detailed wooden toy ship with intricately carved masts and sails is seen gliding smoothly over
	a plush, blue carpet that mimics the waves of the sea. The ship's hull is painted a rich brown,
	with tiny windows. The carpet, soft and textured, provides a perfect backdrop, resembling an
	oceanic expanse. Surrounding the ship are various other toys and children's items, hinting at
	a playful environment. The scene captures the innocence and imagination of childhood,
	with the toy ship's journey symbolizing endless adventures in a whimsical, indoor setting.
	"""

	video = pipeline(
	prompt=prompt,
	guidance_scale=6,
	num_inference_steps=50
	).frames[0]
	export_to_video(video, "output.mp4", fps=8)
	```

	### negative_prompt

	Um prompt negativo é útil para excluir elementos que você não deseja ver no vídeo gerado. Ele é comumente usado para refinar a qualidade e o alinhamento do vídeo gerado, afastando o modelo de elementos indesejados, como "blurry, distorted, ugly". Isso pode resultar em vídeos mais limpos e focados.

	```py
	# pip install ftfy
	import torch
	from diffusers import WanPipeline
	from diffusers.schedulers.scheduling_unipc_multistep import UniPCMultistepScheduler
	from diffusers.utils import export_to_video

	vae = AutoencoderKLWan.from_pretrained(
	"Wan-AI/Wan2.1-T2V-14B-Diffusers", subfolder="vae", dtype=torch.float32
	)
	pipeline = WanPipeline.from_pretrained(
	"Wan-AI/Wan2.1-T2V-14B-Diffusers", vae=vae, dtype=torch.bfloat16
	)
	pipeline.scheduler = UniPCMultistepScheduler.from_config(
	pipeline.scheduler.config, flow_shift=5.0
	)
	pipeline.to("cuda") # or "mps", "xpu", "cpu"

	pipeline.load_lora_weights("benjamin-paine/steamboat-willie-14b", adapter_name="steamboat-willie")
	pipeline.set_adapters("steamboat-willie")

	pipeline.enable_model_cpu_offload()

	# use "steamboat willie style" to trigger the LoRA
	prompt = """
	steamboat willie style, golden era animation, The camera rushes from far to near in a low-angle shot,
	revealing a white ferret on a log. It plays, leaps into the water, and emerges, as the camera zooms in
	for a close-up. Water splashes berry bushes nearby, while moss, snow, and leaves blanket the ground.
	Birch trees and a light blue sky frame the scene, with ferns in the foreground. Side lighting casts
	dynamic shadows and warm highlights. Medium composition, front view, low angle, with depth of field.
	"""

	output = pipeline(
	prompt=prompt,
	num_frames=81,
	guidance_scale=5.0,
	).frames[0]
	export_to_video(output, "output.mp4", fps=16)
	```

	## Reduzir o uso de memória

	Modelos de vídeo recentes, como [HunyuanVideoPipeline](/docs/diffusers/v0.43.0/pt/api/pipelines/hunyuan_video#diffusers.HunyuanVideoPipeline) e [WanPipeline](/docs/diffusers/v0.43.0/pt/api/pipelines/wan#diffusers.WanPipeline), que possuem mais de 10 bilhões de parâmetros, exigem uma grande quantidade de memória, frequentemente excedendo a memória disponível em hardware de consumo. O Diffusers oferece várias técnicas para reduzir os requisitos de memória desses modelos grandes.

	> [!TIP]
	> Consulte o guia [Reduzir o uso de memória](../optimization/memory) para mais detalhes sobre outras técnicas de economia de memória.

	Uma dessas técnicas é o [group-offloading](../optimization/memory#group-offloading), que transfere grupos de camadas internas do modelo (como `torch.nn.Sequential`) para a CPU quando não estão em uso. Essas camadas são carregadas apenas quando são necessárias para o cálculo, evitando armazenar todos os componentes do modelo na GPU. Para um modelo com 14 bilhões de parâmetros, como o [WanPipeline](/docs/diffusers/v0.43.0/pt/api/pipelines/wan#diffusers.WanPipeline), o group-offloading pode reduzir a memória necessária para cerca de 13 GB de VRAM.

	```py
	# pip install ftfy
	import torch
	import numpy as np
	from diffusers import AutoModel, WanPipeline
	from diffusers.hooks.group_offloading import apply_group_offloading
	from diffusers.utils import export_to_video, load_image
	from transformers import UMT5EncoderModel

	text_encoder = UMT5EncoderModel.from_pretrained("Wan-AI/Wan2.1-T2V-14B-Diffusers", subfolder="text_encoder", dtype=torch.bfloat16)
	vae = AutoModel.from_pretrained("Wan-AI/Wan2.1-T2V-14B-Diffusers", subfolder="vae", dtype=torch.float32)
	transformer = AutoModel.from_pretrained("Wan-AI/Wan2.1-T2V-14B-Diffusers", subfolder="transformer", dtype=torch.bfloat16)

	# group-offloading
	onload_device = torch.device("cuda")
	offload_device = torch.device("cpu")
	apply_group_offloading(text_encoder,
	onload_device=onload_device,
	offload_device=offload_device,
	offload_type="block_level",
	num_blocks_per_group=4
	)
	transformer.enable_group_offload(
	onload_device=onload_device,
	offload_device=offload_device,
	offload_type="leaf_level",
	use_stream=True
	)

	pipeline = WanPipeline.from_pretrained(
	"Wan-AI/Wan2.1-T2V-14B-Diffusers",
	vae=vae,
	transformer=transformer,
	text_encoder=text_encoder,
	dtype=torch.bfloat16
	)
	pipeline.to("cuda") # or "mps", "xpu", "cpu"

	prompt = """
	The camera rushes from far to near in a low-angle shot,
	revealing a white ferret on a log. It plays, leaps into the water, and emerges, as the camera zooms in
	for a close-up. Water splashes berry bushes nearby, while moss, snow, and leaves blanket the ground.
	Birch trees and a light blue sky frame the scene, with ferns in the foreground. Side lighting casts dynamic
	shadows and warm highlights. Medium composition, front view, low angle, with depth of field.
	"""
	negative_prompt = """
	Bright tones, overexposed, static, blurred details, subtitles, style, works, paintings, images, static, overall gray, worst quality,
	low quality, JPEG compression residue, ugly, incomplete, extra fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured,
	misshapen limbs, fused fingers, still picture, messy background, three legs, many people in the background, walking backwards
	"""

	output = pipeline(
	prompt=prompt,
	negative_prompt=negative_prompt,
	num_frames=81,
	guidance_scale=5.0,
	).frames[0]
	export_to_video(output, "output.mp4", fps=16)
	```

	Outra opção para reduzir a memória é considerar a quantização de um modelo, que armazena os pesos do modelo em um tipo de dados de menor precisão. No entanto, a quantização pode impactar a qualidade do vídeo, dependendo do modelo de vídeo específico. Consulte a [Visão geral](../quantization/overview) da quantização para saber mais sobre os diferentes backends de quantização suportados.

	O exemplo abaixo usa [bitsandbytes](../quantization/bitsandbytes) para quantizar um modelo.

	```py
	# pip install ftfy

	import torch
	from diffusers import WanPipeline
	from diffusers import AutoModel, WanPipeline
	from diffusers.quantizers import PipelineQuantizationConfig
	from diffusers.schedulers.scheduling_unipc_multistep import UniPCMultistepScheduler
	from transformers import UMT5EncoderModel
	from diffusers.utils import export_to_video

	# quantize transformer and text encoder weights with bitsandbytes
	pipeline_quant_config = PipelineQuantizationConfig(
	quant_backend="bitsandbytes_4bit",
	quant_kwargs={"load_in_4bit": True},
	components_to_quantize=["transformer", "text_encoder"]
	)

	vae = AutoModel.from_pretrained(
	"Wan-AI/Wan2.1-T2V-14B-Diffusers", subfolder="vae", dtype=torch.float32
	)
	pipeline = WanPipeline.from_pretrained(
	"Wan-AI/Wan2.1-T2V-14B-Diffusers", vae=vae, quantization_config=pipeline_quant_config, dtype=torch.bfloat16
	)
	pipeline.scheduler = UniPCMultistepScheduler.from_config(
	pipeline.scheduler.config, flow_shift=5.0
	)
	pipeline.to("cuda") # or "mps", "xpu", "cpu"

	pipeline.load_lora_weights("benjamin-paine/steamboat-willie-14b", adapter_name="steamboat-willie")
	pipeline.set_adapters("steamboat-willie")

	pipeline.enable_model_cpu_offload()

	# use "steamboat willie style" to trigger the LoRA
	prompt = """
	steamboat willie style, golden era animation, The camera rushes from far to near in a low-angle shot,
	revealing a white ferret on a log. It plays, leaps into the water, and emerges, as the camera zooms in
	for a close-up. Water splashes berry bushes nearby, while moss, snow, and leaves blanket the ground.
	Birch trees and a light blue sky frame the scene, with ferns in the foreground. Side lighting casts
	dynamic shadows and warm highlights. Medium composition, front view, low angle, with depth of field.
	"""

	output = pipeline(
	prompt=prompt,
	num_frames=81,
	guidance_scale=5.0,
	).frames[0]
	export_to_video(output, "output.mp4", fps=16)
	```

	## Velocidade de inferência

	[torch.compile](https://docs.pytorch.org/tutorials/intermediate/torch_compile_tutorial.html) pode acelerar a inferência ao usar kernels otimizados. A compilação leva mais tempo na primeira vez, mas, uma vez compilado, é muito mais rápido. O ideal é compilar o pipeline uma vez e, em seguida, usá-lo várias vezes sem alterar nada. Uma alteração, como no tamanho da imagem, dispara a recompilação.

	O exemplo abaixo compila o transformer no pipeline e usa o modo `"max-autotune"` para maximizar o desempenho.

	```py
	import torch
	from diffusers import CogVideoXPipeline, CogVideoXTransformer3DModel
	from diffusers.utils import export_to_video

	pipeline = CogVideoXPipeline.from_pretrained(
	"THUDM/CogVideoX-2b",
	dtype=torch.float16
	).to("cuda") # or "mps", "xpu", "cpu"

	# torch.compile
	pipeline.transformer.to(memory_format=torch.channels_last)
	pipeline.transformer = torch.compile(
	pipeline.transformer, mode="max-autotune", fullgraph=True
	)

	prompt = """
	A detailed wooden toy ship with intricately carved masts and sails is seen gliding smoothly over a plush, blue carpet that mimics the waves of the sea.
	The ship's hull is painted a rich brown, with tiny windows. The carpet, soft and textured, provides a perfect backdrop, resembling an oceanic expanse.
	Surrounding the ship are various other toys and children's items, hinting at a playful environment. The scene captures the innocence and imagination of childhood,
	with the toy ship's journey symbolizing endless adventures in a whimsical, indoor setting.
	"""

	video = pipeline(
	prompt=prompt,
	guidance_scale=6,
	num_inference_steps=50
	).frames[0]
	export_to_video(video, "output.mp4", fps=8)
	```