# Generasi video

Model generasi video memperluas generasi gambar (yang dapat dianggap sebagai video 1-frame) untuk juga memproses data yang berkaitan dengan ruang dan waktu. Memastikan semua data ini - teks, ruang, waktu - tetap konsisten dan selaras dari frame ke frame merupakan tantangan besar dalam menghasilkan video yang panjang dan beresolusi tinggi.

Model video modern mengatasi tantangan ini dengan arsitektur diffusion transformer (DiT). Hal ini mengurangi biaya komputasi dan memungkinkan scaling yang lebih efisien ke data gambar dan video yang lebih besar dan berkualitas lebih tinggi.

Lihat apa yang bisa dilakukan beberapa model video ini di bawah.

```py
# pip install ftfy
import torch
import numpy as np
from diffusers import AutoModel, WanPipeline
from diffusers.hooks.group_offloading import apply_group_offloading
from diffusers.utils import export_to_video, load_image
from transformers import UMT5EncoderModel

text_encoder = UMT5EncoderModel.from_pretrained("Wan-AI/Wan2.1-T2V-14B-Diffusers", subfolder="text_encoder", dtype=torch.bfloat16)
vae = AutoModel.from_pretrained("Wan-AI/Wan2.1-T2V-14B-Diffusers", subfolder="vae", dtype=torch.float32)
transformer = AutoModel.from_pretrained("Wan-AI/Wan2.1-T2V-14B-Diffusers", subfolder="transformer", dtype=torch.bfloat16)

# group-offloading
onload_device = torch.device("cuda")
offload_device = torch.device("cpu")
apply_group_offloading(text_encoder,
    onload_device=onload_device,
    offload_device=offload_device,
    offload_type="block_level",
    num_blocks_per_group=4
)
transformer.enable_group_offload(
    onload_device=onload_device,
    offload_device=offload_device,
    offload_type="leaf_level",
    use_stream=True
)

pipeline = WanPipeline.from_pretrained(
    "Wan-AI/Wan2.1-T2V-14B-Diffusers",
    vae=vae,
    transformer=transformer,
    text_encoder=text_encoder,
    dtype=torch.bfloat16
)
pipeline.to("cuda")  # or "mps", "xpu", "cpu"

prompt = """
The camera rushes from far to near in a low-angle shot, 
revealing a white ferret on a log. It plays, leaps into the water, and emerges, as the camera zooms in 
for a close-up. Water splashes berry bushes nearby, while moss, snow, and leaves blanket the ground. 
Birch trees and a light blue sky frame the scene, with ferns in the foreground. Side lighting casts dynamic 
shadows and warm highlights. Medium composition, front view, low angle, with depth of field.
"""
negative_prompt = """
Bright tones, overexposed, static, blurred details, subtitles, style, works, paintings, images, static, overall gray, worst quality, 
low quality, JPEG compression residue, ugly, incomplete, extra fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured, 
misshapen limbs, fused fingers, still picture, messy background, three legs, many people in the background, walking backwards
"""

output = pipeline(
    prompt=prompt,
    negative_prompt=negative_prompt,
    num_frames=81,
    guidance_scale=5.0,
).frames[0]
export_to_video(output, "output.mp4", fps=16)
```

```py
import torch
from diffusers importAutoModel, HunyuanVideoPipeline
from diffusers.quantizers import PipelineQuantizationConfig
from diffusers.utils import export_to_video

# quantize weights to int4 with bitsandbytes
pipeline_quant_config = PipelineQuantizationConfig(
  quant_backend="bitsandbytes_4bit",
  quant_kwargs={
    "load_in_4bit": True,
    "bnb_4bit_quant_type": "nf4",
    "bnb_4bit_compute_dtype": torch.bfloat16
    },
  components_to_quantize="transformer"
)

pipeline = HunyuanVideoPipeline.from_pretrained(
    "hunyuanvideo-community/HunyuanVideo",
    quantization_config=pipeline_quant_config,
    dtype=torch.bfloat16,
)

# model-offloading and tiling
pipeline.enable_model_cpu_offload()
pipeline.vae.enable_tiling()

prompt = "A fluffy teddy bear sits on a bed of soft pillows surrounded by children's toys."
video = pipeline(prompt=prompt, num_frames=61, num_inference_steps=30).frames[0]
export_to_video(video, "output.mp4", fps=15)
```

```py
import torch
from diffusers import LTXPipeline, AutoModel
from diffusers.hooks import apply_group_offloading
from diffusers.utils import export_to_video

# fp8 layerwise weight-casting
transformer = AutoModel.from_pretrained(
    "Lightricks/LTX-Video",
    subfolder="transformer",
    dtype=torch.bfloat16
)
transformer.enable_layerwise_casting(
    storage_dtype=torch.float8_e4m3fn, compute_dtype=torch.bfloat16
)

pipeline = LTXPipeline.from_pretrained("Lightricks/LTX-Video", transformer=transformer, dtype=torch.bfloat16)

# group-offloading
onload_device = torch.device("cuda")
offload_device = torch.device("cpu")
pipeline.transformer.enable_group_offload(onload_device=onload_device, offload_device=offload_device, offload_type="leaf_level", use_stream=True)
apply_group_offloading(pipeline.text_encoder, onload_device=onload_device, offload_type="block_level", num_blocks_per_group=2)
apply_group_offloading(pipeline.vae, onload_device=onload_device, offload_type="leaf_level")

prompt = """
A woman with long brown hair and light skin smiles at another woman with long blonde hair. The woman with brown hair wears a black jacket and has a small, barely noticeable mole on her right cheek. The camera angle is a close-up, focused on the woman with brown hair's face. The lighting is warm and natural, likely from the setting sun, casting a soft glow on the scene. The scene appears to be real-life footage
"""
negative_prompt = "worst quality, inconsistent motion, blurry, jittery, distorted"

video = pipeline(
    prompt=prompt,
    negative_prompt=negative_prompt,
    width=768,
    height=512,
    num_frames=161,
    decode_timestep=0.03,
    decode_noise_scale=0.025,
    num_inference_steps=50,
).frames[0]
export_to_video(video, "output.mp4", fps=24)
```

Panduan ini akan membahas dasar-dasar pembuatan video, seperti parameter apa yang perlu dikonfigurasi dan bagaimana cara mengurangi penggunaan memori mereka.

> [!TIP]
> Jika Anda tertarik untuk mempelajari lebih lanjut tentang cara menggunakan model tertentu, silakan merujuk pada kartu model API pipeline mereka.

## Parameter Pipeline

Ada beberapa parameter yang perlu dikonfigurasi dalam pipeline yang akan memengaruhi kualitas atau kecepatan generasi video. Eksperimen dengan nilai parameter yang berbeda sangat penting untuk menemukan tradeoff antara kualitas dan kecepatan yang tepat.

### num_frames

Frame adalah gambar diam yang diputar dalam urutan frame lain untuk menciptakan gerakan atau video. Kendalikan jumlah frame yang dihasilkan per detik dengan `num_frames`. Meningkatkan `num_frames` akan meningkatkan kehalusan gerakan yang dipersepsikan dan koherensi visual, sehingga sangat penting untuk video dengan konten dinamis. Nilai `num_frames` yang lebih tinggi juga meningkatkan durasi video.

Beberapa model video memerlukan nilai `num_frames` yang lebih spesifik untuk inferensi. Misalnya, [HunyuanVideoPipeline](/docs/diffusers/v0.43.0/id/api/pipelines/hunyuan_video#diffusers.HunyuanVideoPipeline) merekomendasikan untuk menghitung `num_frames` dengan `(4 * num_frames) +1`. Selalu periksa kartu model API pipeline untuk melihat apakah ada nilai yang direkomendasikan.

```py
import torch
from diffusers import LTXPipeline
from diffusers.utils import export_to_video

pipeline = LTXPipeline.from_pretrained(
    "Lightricks/LTX-Video", dtype=torch.bfloat16
).to("cuda")  # or "mps", "xpu", "cpu"

prompt = """
A woman with long brown hair and light skin smiles at another woman with long blonde hair. The woman 
with brown hair wears a black jacket and has a small, barely noticeable mole on her right cheek. The 
camera angle is a close-up, focused on the woman with brown hair's face. The lighting is warm and 
natural, likely from the setting sun, casting a soft glow on the scene. The scene appears to be 
real-life footage
"""

video = pipeline(
    prompt=prompt,
    negative_prompt=negative_prompt,
    width=768,
    height=512,
    num_frames=161,
    decode_timestep=0.03,
    decode_noise_scale=0.025,
    num_inference_steps=50,
).frames[0]
export_to_video(video, "output.mp4", fps=24)
```

### guidance_scale

Skala panduan atau "cfg" mengontrol seberapa ketat frame yang dihasilkan mengikuti kondisi input (teks, gambar, atau keduanya). Meningkatkan `guidance_scale` menghasilkan frame yang lebih menyerupai kondisi input dan mencakup detail yang lebih halus, namun berisiko memperkenalkan artefak dan mengurangi keragaman output. Nilai `guidance_scale` yang lebih rendah mendorong kepatuhan prompt yang lebih longgar dan meningkatkan variasi output, tetapi detailnya mungkin tidak sebaik sebelumnya. Jika terlalu rendah, model mungkin mengabaikan prompt Anda sepenuhnya dan menghasilkan noise acak.

```py
import torch
from diffusers import CogVideoXPipeline, CogVideoXTransformer3DModel
from diffusers.utils import export_to_video

pipeline = CogVideoXPipeline.from_pretrained(
  "THUDM/CogVideoX-2b",
  dtype=torch.float16
).to("cuda")  # or "mps", "xpu", "cpu"

prompt = """
A detailed wooden toy ship with intricately carved masts and sails is seen gliding smoothly over
a plush, blue carpet that mimics the waves of the sea. The ship's hull is painted a rich brown, 
with tiny windows. The carpet, soft and textured, provides a perfect backdrop, resembling an 
oceanic expanse. Surrounding the ship are various other toys and children's items, hinting at 
a playful environment. The scene captures the innocence and imagination of childhood, 
with the toy ship's journey symbolizing endless adventures in a whimsical, indoor setting.
"""

video = pipeline(
  prompt=prompt,
  guidance_scale=6,
  num_inference_steps=50
).frames[0]
export_to_video(video, "output.mp4", fps=8)
```

### negative_prompt

Prompt negatif berguna untuk mengecualikan hal-hal yang tidak ingin Anda lihat dalam video yang dihasilkan. Hal ini umum digunakan untuk menyempurnakan kualitas dan keselarasan video yang dihasilkan dengan mendorong model menjauh dari elemen-elemen yang tidak diinginkan seperti "blurry, distorted, ugly". Hal ini dapat menghasilkan video yang lebih bersih dan lebih fokus.

```py
# pip install ftfy
import torch
from diffusers import WanPipeline
from diffusers.schedulers.scheduling_unipc_multistep import UniPCMultistepScheduler
from diffusers.utils import export_to_video

vae = AutoencoderKLWan.from_pretrained(
  "Wan-AI/Wan2.1-T2V-14B-Diffusers", subfolder="vae", dtype=torch.float32
)
pipeline = WanPipeline.from_pretrained(
  "Wan-AI/Wan2.1-T2V-14B-Diffusers", vae=vae, dtype=torch.bfloat16
)
pipeline.scheduler = UniPCMultistepScheduler.from_config(
  pipeline.scheduler.config, flow_shift=5.0
)
pipeline.to("cuda")  # or "mps", "xpu", "cpu"

pipeline.load_lora_weights("benjamin-paine/steamboat-willie-14b", adapter_name="steamboat-willie")
pipeline.set_adapters("steamboat-willie")

pipeline.enable_model_cpu_offload()

# use "steamboat willie style" to trigger the LoRA
prompt = """
steamboat willie style, golden era animation, The camera rushes from far to near in a low-angle shot, 
revealing a white ferret on a log. It plays, leaps into the water, and emerges, as the camera zooms in 
for a close-up. Water splashes berry bushes nearby, while moss, snow, and leaves blanket the ground. 
Birch trees and a light blue sky frame the scene, with ferns in the foreground. Side lighting casts 
dynamic shadows and warm highlights. Medium composition, front view, low angle, with depth of field.
"""

output = pipeline(
  prompt=prompt,
  num_frames=81,
  guidance_scale=5.0,
).frames[0]
export_to_video(output, "output.mp4", fps=16)
```

## Kurangi penggunaan memori

Model video terbaru seperti [HunyuanVideoPipeline](/docs/diffusers/v0.43.0/id/api/pipelines/hunyuan_video#diffusers.HunyuanVideoPipeline) dan [WanPipeline](/docs/diffusers/v0.43.0/id/api/pipelines/wan#diffusers.WanPipeline), yang memiliki lebih dari 10 miliar parameter, membutuhkan banyak memori dan sering kali melampaui memori yang tersedia pada perangkat konsumen. Diffusers menawarkan beberapa teknik untuk mengurangi kebutuhan memori dari model-model besar ini.

> [!TIP]
> Lihat panduan [Reduce memory usage](../optimization/memory) untuk detail lebih lanjut tentang teknik penghematan memori lainnya.

Salah satu teknik ini adalah [group-offloading](../optimization/memory#group-offloading), yang memindahkan kelompok lapisan internal model (seperti `torch.nn.Sequential`) ke CPU saat tidak digunakan. Lapisan-lapisan ini hanya dimuat saat diperlukan untuk komputasi guna menghindari penyimpanan **semua** komponen model di GPU. Untuk model dengan 14B parameter seperti [WanPipeline](/docs/diffusers/v0.43.0/id/api/pipelines/wan#diffusers.WanPipeline), group-offloading dapat menurunkan memori yang dibutuhkan menjadi sekitar 13GB VRAM.

```py
# pip install ftfy
import torch
import numpy as np
from diffusers import AutoModel, WanPipeline
from diffusers.hooks.group_offloading import apply_group_offloading
from diffusers.utils import export_to_video, load_image
from transformers import UMT5EncoderModel

text_encoder = UMT5EncoderModel.from_pretrained("Wan-AI/Wan2.1-T2V-14B-Diffusers", subfolder="text_encoder", dtype=torch.bfloat16)
vae = AutoModel.from_pretrained("Wan-AI/Wan2.1-T2V-14B-Diffusers", subfolder="vae", dtype=torch.float32)
transformer = AutoModel.from_pretrained("Wan-AI/Wan2.1-T2V-14B-Diffusers", subfolder="transformer", dtype=torch.bfloat16)

# group-offloading
onload_device = torch.device("cuda")
offload_device = torch.device("cpu")
apply_group_offloading(text_encoder,
    onload_device=onload_device,
    offload_device=offload_device,
    offload_type="block_level",
    num_blocks_per_group=4
)
transformer.enable_group_offload(
    onload_device=onload_device,
    offload_device=offload_device,
    offload_type="leaf_level",
    use_stream=True
)

pipeline = WanPipeline.from_pretrained(
    "Wan-AI/Wan2.1-T2V-14B-Diffusers",
    vae=vae,
    transformer=transformer,
    text_encoder=text_encoder,
    dtype=torch.bfloat16
)
pipeline.to("cuda")  # or "mps", "xpu", "cpu"

prompt = """
The camera rushes from far to near in a low-angle shot, 
revealing a white ferret on a log. It plays, leaps into the water, and emerges, as the camera zooms in 
for a close-up. Water splashes berry bushes nearby, while moss, snow, and leaves blanket the ground. 
Birch trees and a light blue sky frame the scene, with ferns in the foreground. Side lighting casts dynamic 
shadows and warm highlights. Medium composition, front view, low angle, with depth of field.
"""
negative_prompt = """
Bright tones, overexposed, static, blurred details, subtitles, style, works, paintings, images, static, overall gray, worst quality, 
low quality, JPEG compression residue, ugly, incomplete, extra fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured, 
misshapen limbs, fused fingers, still picture, messy background, three legs, many people in the background, walking backwards
"""

output = pipeline(
    prompt=prompt,
    negative_prompt=negative_prompt,
    num_frames=81,
    guidance_scale=5.0,
).frames[0]
export_to_video(output, "output.mp4", fps=16)
```

Opsi lain untuk mengurangi memori adalah mempertimbangkan kuantisasi model, yang menyimpan bobot model dalam tipe data presisi yang lebih rendah. Namun, kuantisasi dapat memengaruhi kualitas video tergantung pada model video tertentu. Lihat [Ikhtisar](../quantization/overview) kuantisasi untuk mempelajari lebih lanjut tentang berbagai backend kuantisasi yang didukung.

Contoh di bawah menggunakan [bitsandbytes](../quantization/bitsandbytes) untuk mengkuantisasi model.

```py
# pip install ftfy

import torch
from diffusers import WanPipeline
from diffusers import AutoModel, WanPipeline
from diffusers.quantizers import PipelineQuantizationConfig
from diffusers.schedulers.scheduling_unipc_multistep import UniPCMultistepScheduler
from transformers import UMT5EncoderModel
from diffusers.utils import export_to_video

# quantize transformer and text encoder weights with bitsandbytes
pipeline_quant_config = PipelineQuantizationConfig(
  quant_backend="bitsandbytes_4bit",
  quant_kwargs={"load_in_4bit": True},
  components_to_quantize=["transformer", "text_encoder"]
)

vae = AutoModel.from_pretrained(
  "Wan-AI/Wan2.1-T2V-14B-Diffusers", subfolder="vae", dtype=torch.float32
)
pipeline = WanPipeline.from_pretrained(
  "Wan-AI/Wan2.1-T2V-14B-Diffusers", vae=vae, quantization_config=pipeline_quant_config, dtype=torch.bfloat16
)
pipeline.scheduler = UniPCMultistepScheduler.from_config(
  pipeline.scheduler.config, flow_shift=5.0
)
pipeline.to("cuda")  # or "mps", "xpu", "cpu"

pipeline.load_lora_weights("benjamin-paine/steamboat-willie-14b", adapter_name="steamboat-willie")
pipeline.set_adapters("steamboat-willie")

pipeline.enable_model_cpu_offload()

# use "steamboat willie style" to trigger the LoRA
prompt = """
steamboat willie style, golden era animation, The camera rushes from far to near in a low-angle shot, 
revealing a white ferret on a log. It plays, leaps into the water, and emerges, as the camera zooms in 
for a close-up. Water splashes berry bushes nearby, while moss, snow, and leaves blanket the ground. 
Birch trees and a light blue sky frame the scene, with ferns in the foreground. Side lighting casts 
dynamic shadows and warm highlights. Medium composition, front view, low angle, with depth of field.
"""

output = pipeline(
  prompt=prompt,
  num_frames=81,
  guidance_scale=5.0,
).frames[0]
export_to_video(output, "output.mp4", fps=16)
```

## Kecepatan inferensi

[torch.compile](https://docs.pytorch.org/tutorials/intermediate/torch_compile_tutorial.html) dapat mempercepat inferensi dengan menggunakan kernel yang dioptimalkan. Kompilasi memakan waktu lebih lama pada penggunaan pertama, tetapi setelah dikompilasi, prosesnya menjadi jauh lebih cepat. Sebaiknya kompilasi pipeline hanya sekali, lalu gunakan pipeline tersebut beberapa kali tanpa mengubah apa pun. Perubahan, seperti pada ukuran gambar, akan memicu kompilasi ulang.

Contoh di bawah ini mengompilasi transformer dalam pipeline dan menggunakan mode `"max-autotune"` untuk memaksimalkan kinerja.

```py
import torch
from diffusers import CogVideoXPipeline, CogVideoXTransformer3DModel
from diffusers.utils import export_to_video

pipeline = CogVideoXPipeline.from_pretrained(
  "THUDM/CogVideoX-2b",
  dtype=torch.float16
).to("cuda")  # or "mps", "xpu", "cpu"

# torch.compile
pipeline.transformer.to(memory_format=torch.channels_last)
pipeline.transformer = torch.compile(
    pipeline.transformer, mode="max-autotune", fullgraph=True
)

prompt = """
A detailed wooden toy ship with intricately carved masts and sails is seen gliding smoothly over a plush, blue carpet that mimics the waves of the sea. 
The ship's hull is painted a rich brown, with tiny windows. The carpet, soft and textured, provides a perfect backdrop, resembling an oceanic expanse. 
Surrounding the ship are various other toys and children's items, hinting at a playful environment. The scene captures the innocence and imagination of childhood, 
with the toy ship's journey symbolizing endless adventures in a whimsical, indoor setting.
"""

video = pipeline(
  prompt=prompt,
  guidance_scale=6,
  num_inference_steps=50
).frames[0]
export_to_video(video, "output.mp4", fps=8)
```