Buckets:

hf-doc-build
/

doc

Files
xet
hf-doc-build/doc / diffusers /main /ru /using-diffusers /text-img2vid.html
mishig
3 days ago
download
raw
58.8 kB
<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;Генерация видео&quot;,&quot;local&quot;:&quot;video-generation&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Параметры конвейера&quot;,&quot;local&quot;:&quot;pipeline-parameters&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;num_frames&quot;,&quot;local&quot;:&quot;numframes&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;guidance_scale&quot;,&quot;local&quot;:&quot;guidancescale&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;negative_prompt&quot;,&quot;local&quot;:&quot;negativeprompt&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Сокращение использования памяти&quot;,&quot;local&quot;:&quot;reduce-memory-usage&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Скорость инференса&quot;,&quot;local&quot;:&quot;inference-speed&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}"/>
		<link href="/docs/diffusers/main/en/_app/immutable/entry/start.0S0_iEoG.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/I9eooN77.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/Bupkjr9f.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/entry/app.CPdOxCbw.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/CyvZ3FS5.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/l0QnnOED.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/DnVIJfC5.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/DsnmJJEf.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/nodes/0.DteTvmQZ.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/CoSrdQax.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/nodes/407.BqN650YR.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/DSPDl1gz.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/D5eijdnr.js" rel="modulepreload">
		<!--g33fzy--><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;Video generation&quot;,&quot;local&quot;:&quot;video-generation&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Pipeline parameters&quot;,&quot;local&quot;:&quot;pipeline-parameters&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;num_frames&quot;,&quot;local&quot;:&quot;numframes&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;guidance_scale&quot;,&quot;local&quot;:&quot;guidancescale&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;negative_prompt&quot;,&quot;local&quot;:&quot;negativeprompt&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Reduce memory usage&quot;,&quot;local&quot;:&quot;reduce-memory-usage&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Inference speed&quot;,&quot;local&quot;:&quot;inference-speed&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}"/><!---->
		<link href="/docs/diffusers/main/en/_app/immutable/assets/0.tn0RQdqM.css" rel="modulepreload"> <!--[--><!--[0--><!--[--><!--[0--><!--[--><p></p> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: inline-end; margin-inline-start: 10px; display: inline-flex; position: relative; z-index: 10;"><div dir="ltr" class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 h-7 max-sm:h-7 px-2 max-sm:px-1.5 text-sm font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0 hover:text-gray-800 dark:hover:text-gray-200"><svg class="sm:size-3.5 size-3" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----></span> <span dir="auto">Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-7 max-sm:h-7 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible sm:size-3.5 size-3 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg><!----></button></div> <!--[-1--><!--]--></div><!----> <!--[0--><h1 class="relative group"><a id="video-generation" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#video-generation"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Генерация видео<!----></span></h1><!--]--><!----> <p>Модели генерации видео расширяют возможности генерации изображений (которую можно рассматривать как видео из одного кадра), добавляя обработку данных, связанных с пространством и временем. Обеспечение согласованности и выравнивания всех этих данных — текста, пространства и времени — от кадра к кадру представляет собой серьезную задачу при генерации длинных видео высокого разрешения.</p> <p>Современные модели для работы с видео решают эту задачу с помощью архитектуры диффузионного трансформера (DiT). Это снижает вычислительные затраты и позволяет более эффективно масштабироваться на более крупные и качественные данные изображений и видео.</p> <p>Ниже вы можете посмотреть, на что способны некоторые из этих видео-моделей.</p> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"><!--[--><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd border-gray-800 bg-black dark:bg-gray-700 text-white">Wan2.1</div><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd text-gray-500 cursor-pointer opacity-90 hover:text-gray-700 dark:hover:text-gray-200 hover:shadow-sm">HunyuanVideo</div><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd text-gray-500 cursor-pointer opacity-90 hover:text-gray-700 dark:hover:text-gray-200 hover:shadow-sm">LTX-Video</div><!--]--></div> <div class="language-select"><div><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Скопировано</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-comment"># pip install ftfy</span>
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> AutoModel, WanPipeline
<span class="hljs-keyword">from</span> diffusers.hooks.group_offloading <span class="hljs-keyword">import</span> apply_group_offloading
<span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video, load_image
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> UMT5EncoderModel

text_encoder = UMT5EncoderModel.from_pretrained(<span class="hljs-string">&quot;Wan-AI/Wan2.1-T2V-14B-Diffusers&quot;</span>, subfolder=<span class="hljs-string">&quot;text_encoder&quot;</span>, dtype=torch.bfloat16)
vae = AutoModel.from_pretrained(<span class="hljs-string">&quot;Wan-AI/Wan2.1-T2V-14B-Diffusers&quot;</span>, subfolder=<span class="hljs-string">&quot;vae&quot;</span>, dtype=torch.float32)
transformer = AutoModel.from_pretrained(<span class="hljs-string">&quot;Wan-AI/Wan2.1-T2V-14B-Diffusers&quot;</span>, subfolder=<span class="hljs-string">&quot;transformer&quot;</span>, dtype=torch.bfloat16)

<span class="hljs-comment"># group-offloading</span>
onload_device = torch.device(<span class="hljs-string">&quot;cuda&quot;</span>)
offload_device = torch.device(<span class="hljs-string">&quot;cpu&quot;</span>)
apply_group_offloading(text_encoder,
    onload_device=onload_device,
    offload_device=offload_device,
    offload_type=<span class="hljs-string">&quot;block_level&quot;</span>,
    num_blocks_per_group=<span class="hljs-number">4</span>
)
transformer.enable_group_offload(
    onload_device=onload_device,
    offload_device=offload_device,
    offload_type=<span class="hljs-string">&quot;leaf_level&quot;</span>,
    use_stream=<span class="hljs-literal">True</span>
)

pipeline = WanPipeline.from_pretrained(
    <span class="hljs-string">&quot;Wan-AI/Wan2.1-T2V-14B-Diffusers&quot;</span>,
    vae=vae,
    transformer=transformer,
    text_encoder=text_encoder,
    dtype=torch.bfloat16
)
pipeline.to(<span class="hljs-string">&quot;cuda&quot;</span>)  <span class="hljs-comment"># or &quot;mps&quot;, &quot;xpu&quot;, &quot;cpu&quot;</span>

prompt = <span class="hljs-string">&quot;&quot;&quot;
The camera rushes from far to near in a low-angle shot, 
revealing a white ferret on a log. It plays, leaps into the water, and emerges, as the camera zooms in 
for a close-up. Water splashes berry bushes nearby, while moss, snow, and leaves blanket the ground. 
Birch trees and a light blue sky frame the scene, with ferns in the foreground. Side lighting casts dynamic 
shadows and warm highlights. Medium composition, front view, low angle, with depth of field.
&quot;&quot;&quot;</span>
negative_prompt = <span class="hljs-string">&quot;&quot;&quot;
Bright tones, overexposed, static, blurred details, subtitles, style, works, paintings, images, static, overall gray, worst quality, 
low quality, JPEG compression residue, ugly, incomplete, extra fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured, 
misshapen limbs, fused fingers, still picture, messy background, three legs, many people in the background, walking backwards
&quot;&quot;&quot;</span>

output = pipeline(
    prompt=prompt,
    negative_prompt=negative_prompt,
    num_frames=<span class="hljs-number">81</span>,
    guidance_scale=<span class="hljs-number">5.0</span>,
).frames[<span class="hljs-number">0</span>]
export_to_video(output, <span class="hljs-string">&quot;output.mp4&quot;</span>, fps=<span class="hljs-number">16</span>)<!----></pre></div><!----></div><!----> <div hidden=""><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Скопировано</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers importAutoModel, HunyuanVideoPipeline
<span class="hljs-keyword">from</span> diffusers.quantizers <span class="hljs-keyword">import</span> PipelineQuantizationConfig
<span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video

<span class="hljs-comment"># quantize weights to int4 with bitsandbytes</span>
pipeline_quant_config = PipelineQuantizationConfig(
  quant_backend=<span class="hljs-string">&quot;bitsandbytes_4bit&quot;</span>,
  quant_kwargs={
    <span class="hljs-string">&quot;load_in_4bit&quot;</span>: <span class="hljs-literal">True</span>,
    <span class="hljs-string">&quot;bnb_4bit_quant_type&quot;</span>: <span class="hljs-string">&quot;nf4&quot;</span>,
    <span class="hljs-string">&quot;bnb_4bit_compute_dtype&quot;</span>: torch.bfloat16
    },
  components_to_quantize=<span class="hljs-string">&quot;transformer&quot;</span>
)

pipeline = HunyuanVideoPipeline.from_pretrained(
    <span class="hljs-string">&quot;hunyuanvideo-community/HunyuanVideo&quot;</span>,
    quantization_config=pipeline_quant_config,
    dtype=torch.bfloat16,
)

<span class="hljs-comment"># model-offloading and tiling</span>
pipeline.enable_model_cpu_offload()
pipeline.vae.enable_tiling()

prompt = <span class="hljs-string">&quot;A fluffy teddy bear sits on a bed of soft pillows surrounded by children&#x27;s toys.&quot;</span>
video = pipeline(prompt=prompt, num_frames=<span class="hljs-number">61</span>, num_inference_steps=<span class="hljs-number">30</span>).frames[<span class="hljs-number">0</span>]
export_to_video(video, <span class="hljs-string">&quot;output.mp4&quot;</span>, fps=<span class="hljs-number">15</span>)<!----></pre></div><!----></div><!----> <div hidden=""><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Скопировано</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> LTXPipeline, AutoModel
<span class="hljs-keyword">from</span> diffusers.hooks <span class="hljs-keyword">import</span> apply_group_offloading
<span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video

<span class="hljs-comment"># fp8 layerwise weight-casting</span>
transformer = AutoModel.from_pretrained(
    <span class="hljs-string">&quot;Lightricks/LTX-Video&quot;</span>,
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    dtype=torch.bfloat16
)
transformer.enable_layerwise_casting(
    storage_dtype=torch.float8_e4m3fn, compute_dtype=torch.bfloat16
)

pipeline = LTXPipeline.from_pretrained(<span class="hljs-string">&quot;Lightricks/LTX-Video&quot;</span>, transformer=transformer, dtype=torch.bfloat16)

<span class="hljs-comment"># group-offloading</span>
onload_device = torch.device(<span class="hljs-string">&quot;cuda&quot;</span>)
offload_device = torch.device(<span class="hljs-string">&quot;cpu&quot;</span>)
pipeline.transformer.enable_group_offload(onload_device=onload_device, offload_device=offload_device, offload_type=<span class="hljs-string">&quot;leaf_level&quot;</span>, use_stream=<span class="hljs-literal">True</span>)
apply_group_offloading(pipeline.text_encoder, onload_device=onload_device, offload_type=<span class="hljs-string">&quot;block_level&quot;</span>, num_blocks_per_group=<span class="hljs-number">2</span>)
apply_group_offloading(pipeline.vae, onload_device=onload_device, offload_type=<span class="hljs-string">&quot;leaf_level&quot;</span>)

prompt = <span class="hljs-string">&quot;&quot;&quot;
A woman with long brown hair and light skin smiles at another woman with long blonde hair. The woman with brown hair wears a black jacket and has a small, barely noticeable mole on her right cheek. The camera angle is a close-up, focused on the woman with brown hair&#x27;s face. The lighting is warm and natural, likely from the setting sun, casting a soft glow on the scene. The scene appears to be real-life footage
&quot;&quot;&quot;</span>
negative_prompt = <span class="hljs-string">&quot;worst quality, inconsistent motion, blurry, jittery, distorted&quot;</span>

video = pipeline(
    prompt=prompt,
    negative_prompt=negative_prompt,
    width=<span class="hljs-number">768</span>,
    height=<span class="hljs-number">512</span>,
    num_frames=<span class="hljs-number">161</span>,
    decode_timestep=<span class="hljs-number">0.03</span>,
    decode_noise_scale=<span class="hljs-number">0.025</span>,
    num_inference_steps=<span class="hljs-number">50</span>,
).frames[<span class="hljs-number">0</span>]
export_to_video(video, <span class="hljs-string">&quot;output.mp4&quot;</span>, fps=<span class="hljs-number">24</span>)<!----></pre></div><!----></div><!----><!----></div><!----> <p>В этом руководстве рассматриваются основы генерации видео, такие как параметры, которые необходимо настроить, и способы снижения потребления памяти.</p> <blockquote class="tip"><p>Если вы хотите узнать больше о том, как использовать конкретную модель, обратитесь к её карточке модели в API конвейера.</p></blockquote> <!--[1--><h2 class="relative group"><a id="pipeline-parameters" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#pipeline-parameters"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Параметры конвейера<!----></span></h2><!--]--><!----> <p>В пайплайне есть несколько параметров, влияющих на качество или скорость генерации видео. Эксперименты с различными значениями параметров важны для поиска оптимального баланса между качеством и скоростью.</p> <!--[2--><h3 class="relative group"><a id="numframes" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#numframes"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->num_frames<!----></span></h3><!--]--><!----> <p>Кадр — это статичное изображение, которое воспроизводится в последовательности с другими кадрами для создания движения или видео. Количество кадров, генерируемых в секунду, настраивается с помощью <code>num_frames</code>. Увеличение <code>num_frames</code> повышает плавность воспринимаемого движения и визуальную согласованность, что особенно важно для видео с динамичным содержанием. Более высокое значение <code>num_frames</code> также увеличивает длительность видео.</p> <p>Некоторые видеомодели требуют более конкретных значений <code>num_frames</code> для инференса. Например, <a href="/docs/diffusers/main/ru/api/pipelines/hunyuan_video#diffusers.HunyuanVideoPipeline">HunyuanVideoPipeline</a> рекомендует вычислять <code>num_frames</code> по формуле <code>(4 * num_frames) +1</code>. Всегда проверяйте карточку модели пайплайна в API, чтобы узнать, есть ли рекомендуемое значение.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Скопировано</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> LTXPipeline
<span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video

pipeline = LTXPipeline.from_pretrained(
    <span class="hljs-string">&quot;Lightricks/LTX-Video&quot;</span>, dtype=torch.bfloat16
).to(<span class="hljs-string">&quot;cuda&quot;</span>)  <span class="hljs-comment"># or &quot;mps&quot;, &quot;xpu&quot;, &quot;cpu&quot;</span>

prompt = <span class="hljs-string">&quot;&quot;&quot;
A woman with long brown hair and light skin smiles at another woman with long blonde hair. The woman 
with brown hair wears a black jacket and has a small, barely noticeable mole on her right cheek. The 
camera angle is a close-up, focused on the woman with brown hair&#x27;s face. The lighting is warm and 
natural, likely from the setting sun, casting a soft glow on the scene. The scene appears to be 
real-life footage
&quot;&quot;&quot;</span>

video = pipeline(
    prompt=prompt,
    negative_prompt=negative_prompt,
    width=<span class="hljs-number">768</span>,
    height=<span class="hljs-number">512</span>,
    num_frames=<span class="hljs-number">161</span>,
    decode_timestep=<span class="hljs-number">0.03</span>,
    decode_noise_scale=<span class="hljs-number">0.025</span>,
    num_inference_steps=<span class="hljs-number">50</span>,
).frames[<span class="hljs-number">0</span>]
export_to_video(video, <span class="hljs-string">&quot;output.mp4&quot;</span>, fps=<span class="hljs-number">24</span>)<!----></pre></div><!----> <!--[2--><h3 class="relative group"><a id="guidancescale" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#guidancescale"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->guidance_scale<!----></span></h3><!--]--><!----> <p>Параметр Guidance scale или «cfg» управляет тем, насколько сгенерированные кадры соответствуют входным условиям (тексту, изображению или обоим). Увеличение <code>guidance_scale</code> приводит к созданию кадров, которые точнее повторяют входные условия и содержат более мелкие детали, но повышает риск появления артефактов и снижает разнообразие результата. Более низкие значения <code>guidance_scale</code> способствуют более свободному следованию промпту и увеличению разнообразия вывода, однако детали могут быть менее проработанными. Если значение слишком низкое, модель может полностью игнорировать промпт и генерировать случайный шум.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Скопировано</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> CogVideoXPipeline, CogVideoXTransformer3DModel
<span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video

pipeline = CogVideoXPipeline.from_pretrained(
  <span class="hljs-string">&quot;THUDM/CogVideoX-2b&quot;</span>,
  dtype=torch.float16
).to(<span class="hljs-string">&quot;cuda&quot;</span>)  <span class="hljs-comment"># or &quot;mps&quot;, &quot;xpu&quot;, &quot;cpu&quot;</span>

prompt = <span class="hljs-string">&quot;&quot;&quot;
A detailed wooden toy ship with intricately carved masts and sails is seen gliding smoothly over
a plush, blue carpet that mimics the waves of the sea. The ship&#x27;s hull is painted a rich brown, 
with tiny windows. The carpet, soft and textured, provides a perfect backdrop, resembling an 
oceanic expanse. Surrounding the ship are various other toys and children&#x27;s items, hinting at 
a playful environment. The scene captures the innocence and imagination of childhood, 
with the toy ship&#x27;s journey symbolizing endless adventures in a whimsical, indoor setting.
&quot;&quot;&quot;</span>

video = pipeline(
  prompt=prompt,
  guidance_scale=<span class="hljs-number">6</span>,
  num_inference_steps=<span class="hljs-number">50</span>
).frames[<span class="hljs-number">0</span>]
export_to_video(video, <span class="hljs-string">&quot;output.mp4&quot;</span>, fps=<span class="hljs-number">8</span>)<!----></pre></div><!----> <!--[2--><h3 class="relative group"><a id="negativeprompt" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#negativeprompt"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->negative_prompt<!----></span></h3><!--]--><!----> <p>Негативный промпт полезен для исключения элементов, которые вы не хотите видеть в сгенерированном видео. Его часто используют для улучшения качества и соответствия сгенерированного видео, отталкивая модель от нежелательных элементов, таких как «размытое, искажённое, уродливое». Это позволяет создавать более чистые и сфокусированные видео.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Скопировано</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-comment"># pip install ftfy</span>
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> WanPipeline
<span class="hljs-keyword">from</span> diffusers.schedulers.scheduling_unipc_multistep <span class="hljs-keyword">import</span> UniPCMultistepScheduler
<span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video

vae = AutoencoderKLWan.from_pretrained(
  <span class="hljs-string">&quot;Wan-AI/Wan2.1-T2V-14B-Diffusers&quot;</span>, subfolder=<span class="hljs-string">&quot;vae&quot;</span>, dtype=torch.float32
)
pipeline = WanPipeline.from_pretrained(
  <span class="hljs-string">&quot;Wan-AI/Wan2.1-T2V-14B-Diffusers&quot;</span>, vae=vae, dtype=torch.bfloat16
)
pipeline.scheduler = UniPCMultistepScheduler.from_config(
  pipeline.scheduler.config, flow_shift=<span class="hljs-number">5.0</span>
)
pipeline.to(<span class="hljs-string">&quot;cuda&quot;</span>)  <span class="hljs-comment"># or &quot;mps&quot;, &quot;xpu&quot;, &quot;cpu&quot;</span>

pipeline.load_lora_weights(<span class="hljs-string">&quot;benjamin-paine/steamboat-willie-14b&quot;</span>, adapter_name=<span class="hljs-string">&quot;steamboat-willie&quot;</span>)
pipeline.set_adapters(<span class="hljs-string">&quot;steamboat-willie&quot;</span>)

pipeline.enable_model_cpu_offload()

<span class="hljs-comment"># use &quot;steamboat willie style&quot; to trigger the LoRA</span>
prompt = <span class="hljs-string">&quot;&quot;&quot;
steamboat willie style, golden era animation, The camera rushes from far to near in a low-angle shot, 
revealing a white ferret on a log. It plays, leaps into the water, and emerges, as the camera zooms in 
for a close-up. Water splashes berry bushes nearby, while moss, snow, and leaves blanket the ground. 
Birch trees and a light blue sky frame the scene, with ferns in the foreground. Side lighting casts 
dynamic shadows and warm highlights. Medium composition, front view, low angle, with depth of field.
&quot;&quot;&quot;</span>

output = pipeline(
  prompt=prompt,
  num_frames=<span class="hljs-number">81</span>,
  guidance_scale=<span class="hljs-number">5.0</span>,
).frames[<span class="hljs-number">0</span>]
export_to_video(output, <span class="hljs-string">&quot;output.mp4&quot;</span>, fps=<span class="hljs-number">16</span>)<!----></pre></div><!----> <!--[1--><h2 class="relative group"><a id="reduce-memory-usage" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#reduce-memory-usage"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Сокращение использования памяти<!----></span></h2><!--]--><!----> <p>Современные видеомодели, такие как <a href="/docs/diffusers/main/ru/api/pipelines/hunyuan_video#diffusers.HunyuanVideoPipeline">HunyuanVideoPipeline</a> и <a href="/docs/diffusers/main/ru/api/pipelines/wan#diffusers.WanPipeline">WanPipeline</a>, содержащие более 10 млрд параметров, требуют значительного объёма памяти, который часто превышает доступную память на потребительском оборудовании. Diffusers предлагает несколько техник для снижения требований к памяти при работе с такими крупными моделями.</p> <blockquote class="tip"><p>См. руководство <a href="../optimization/memory">Сокращение использования памяти</a> для получения подробной информации о других техниках экономии памяти.</p></blockquote> <p>Одной из таких техник является <a href="../optimization/memory#group-offloading">group-offloading</a>, которая выгружает группы внутренних слоёв модели (например, <code>torch.nn.Sequential</code>) на CPU, когда они не используются. Эти слои загружаются только тогда, когда они нужны для вычислений, чтобы избежать хранения <strong>всех</strong> компонентов модели на GPU. Для модели с 14 млрд параметров, такой как <a href="/docs/diffusers/main/ru/api/pipelines/wan#diffusers.WanPipeline">WanPipeline</a>, group-offloading может снизить требуемый объём памяти до ~13 ГБ VRAM.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Скопировано</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-comment"># pip install ftfy</span>
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> AutoModel, WanPipeline
<span class="hljs-keyword">from</span> diffusers.hooks.group_offloading <span class="hljs-keyword">import</span> apply_group_offloading
<span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video, load_image
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> UMT5EncoderModel

text_encoder = UMT5EncoderModel.from_pretrained(<span class="hljs-string">&quot;Wan-AI/Wan2.1-T2V-14B-Diffusers&quot;</span>, subfolder=<span class="hljs-string">&quot;text_encoder&quot;</span>, dtype=torch.bfloat16)
vae = AutoModel.from_pretrained(<span class="hljs-string">&quot;Wan-AI/Wan2.1-T2V-14B-Diffusers&quot;</span>, subfolder=<span class="hljs-string">&quot;vae&quot;</span>, dtype=torch.float32)
transformer = AutoModel.from_pretrained(<span class="hljs-string">&quot;Wan-AI/Wan2.1-T2V-14B-Diffusers&quot;</span>, subfolder=<span class="hljs-string">&quot;transformer&quot;</span>, dtype=torch.bfloat16)

<span class="hljs-comment"># group-offloading</span>
onload_device = torch.device(<span class="hljs-string">&quot;cuda&quot;</span>)
offload_device = torch.device(<span class="hljs-string">&quot;cpu&quot;</span>)
apply_group_offloading(text_encoder,
    onload_device=onload_device,
    offload_device=offload_device,
    offload_type=<span class="hljs-string">&quot;block_level&quot;</span>,
    num_blocks_per_group=<span class="hljs-number">4</span>
)
transformer.enable_group_offload(
    onload_device=onload_device,
    offload_device=offload_device,
    offload_type=<span class="hljs-string">&quot;leaf_level&quot;</span>,
    use_stream=<span class="hljs-literal">True</span>
)

pipeline = WanPipeline.from_pretrained(
    <span class="hljs-string">&quot;Wan-AI/Wan2.1-T2V-14B-Diffusers&quot;</span>,
    vae=vae,
    transformer=transformer,
    text_encoder=text_encoder,
    dtype=torch.bfloat16
)
pipeline.to(<span class="hljs-string">&quot;cuda&quot;</span>)  <span class="hljs-comment"># or &quot;mps&quot;, &quot;xpu&quot;, &quot;cpu&quot;</span>

prompt = <span class="hljs-string">&quot;&quot;&quot;
The camera rushes from far to near in a low-angle shot, 
revealing a white ferret on a log. It plays, leaps into the water, and emerges, as the camera zooms in 
for a close-up. Water splashes berry bushes nearby, while moss, snow, and leaves blanket the ground. 
Birch trees and a light blue sky frame the scene, with ferns in the foreground. Side lighting casts dynamic 
shadows and warm highlights. Medium composition, front view, low angle, with depth of field.
&quot;&quot;&quot;</span>
negative_prompt = <span class="hljs-string">&quot;&quot;&quot;
Bright tones, overexposed, static, blurred details, subtitles, style, works, paintings, images, static, overall gray, worst quality, 
low quality, JPEG compression residue, ugly, incomplete, extra fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured, 
misshapen limbs, fused fingers, still picture, messy background, three legs, many people in the background, walking backwards
&quot;&quot;&quot;</span>

output = pipeline(
    prompt=prompt,
    negative_prompt=negative_prompt,
    num_frames=<span class="hljs-number">81</span>,
    guidance_scale=<span class="hljs-number">5.0</span>,
).frames[<span class="hljs-number">0</span>]
export_to_video(output, <span class="hljs-string">&quot;output.mp4&quot;</span>, fps=<span class="hljs-number">16</span>)<!----></pre></div><!----> <p>Ещё один способ снизить потребление памяти — квантование модели, при котором веса модели хранятся в формате с пониженной точностью. Однако квантование может повлиять на качество видео в зависимости от конкретной видео-модели. Подробнее о поддерживаемых бэкендах квантования см. в разделе <a href="../quantization/overview">Обзор</a>.</p> <p>Пример ниже использует <a href="../quantization/bitsandbytes">bitsandbytes</a> для квантования модели.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Скопировано</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-comment"># pip install ftfy</span>

<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> WanPipeline
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> AutoModel, WanPipeline
<span class="hljs-keyword">from</span> diffusers.quantizers <span class="hljs-keyword">import</span> PipelineQuantizationConfig
<span class="hljs-keyword">from</span> diffusers.schedulers.scheduling_unipc_multistep <span class="hljs-keyword">import</span> UniPCMultistepScheduler
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> UMT5EncoderModel
<span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video

<span class="hljs-comment"># quantize transformer and text encoder weights with bitsandbytes</span>
pipeline_quant_config = PipelineQuantizationConfig(
  quant_backend=<span class="hljs-string">&quot;bitsandbytes_4bit&quot;</span>,
  quant_kwargs={<span class="hljs-string">&quot;load_in_4bit&quot;</span>: <span class="hljs-literal">True</span>},
  components_to_quantize=[<span class="hljs-string">&quot;transformer&quot;</span>, <span class="hljs-string">&quot;text_encoder&quot;</span>]
)

vae = AutoModel.from_pretrained(
  <span class="hljs-string">&quot;Wan-AI/Wan2.1-T2V-14B-Diffusers&quot;</span>, subfolder=<span class="hljs-string">&quot;vae&quot;</span>, dtype=torch.float32
)
pipeline = WanPipeline.from_pretrained(
  <span class="hljs-string">&quot;Wan-AI/Wan2.1-T2V-14B-Diffusers&quot;</span>, vae=vae, quantization_config=pipeline_quant_config, dtype=torch.bfloat16
)
pipeline.scheduler = UniPCMultistepScheduler.from_config(
  pipeline.scheduler.config, flow_shift=<span class="hljs-number">5.0</span>
)
pipeline.to(<span class="hljs-string">&quot;cuda&quot;</span>)  <span class="hljs-comment"># or &quot;mps&quot;, &quot;xpu&quot;, &quot;cpu&quot;</span>

pipeline.load_lora_weights(<span class="hljs-string">&quot;benjamin-paine/steamboat-willie-14b&quot;</span>, adapter_name=<span class="hljs-string">&quot;steamboat-willie&quot;</span>)
pipeline.set_adapters(<span class="hljs-string">&quot;steamboat-willie&quot;</span>)

pipeline.enable_model_cpu_offload()

<span class="hljs-comment"># use &quot;steamboat willie style&quot; to trigger the LoRA</span>
prompt = <span class="hljs-string">&quot;&quot;&quot;
steamboat willie style, golden era animation, The camera rushes from far to near in a low-angle shot, 
revealing a white ferret on a log. It plays, leaps into the water, and emerges, as the camera zooms in 
for a close-up. Water splashes berry bushes nearby, while moss, snow, and leaves blanket the ground. 
Birch trees and a light blue sky frame the scene, with ferns in the foreground. Side lighting casts 
dynamic shadows and warm highlights. Medium composition, front view, low angle, with depth of field.
&quot;&quot;&quot;</span>

output = pipeline(
  prompt=prompt,
  num_frames=<span class="hljs-number">81</span>,
  guidance_scale=<span class="hljs-number">5.0</span>,
).frames[<span class="hljs-number">0</span>]
export_to_video(output, <span class="hljs-string">&quot;output.mp4&quot;</span>, fps=<span class="hljs-number">16</span>)<!----></pre></div><!----> <!--[1--><h2 class="relative group"><a id="inference-speed" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#inference-speed"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Скорость инференса<!----></span></h2><!--]--><!----> <p><a href="https://docs.pytorch.org/tutorials/intermediate/torch_compile_tutorial.html" rel="nofollow">torch.compile</a> может ускорить инференс за счёт использования оптимизированных ядер. Компиляция занимает больше времени при первом запуске, но после компиляции выполнение становится значительно быстрее. Лучше всего скомпилировать пайплайн один раз, а затем использовать его многократно без каких-либо изменений. Любое изменение, например размера изображения, приводит к повторной компиляции.</p> <p>Пример ниже компилирует трансформер в конвейере и использует режим <code>"max-autotune"</code> для максимальной производительности.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Скопировано</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> CogVideoXPipeline, CogVideoXTransformer3DModel
<span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video

pipeline = CogVideoXPipeline.from_pretrained(
  <span class="hljs-string">&quot;THUDM/CogVideoX-2b&quot;</span>,
  dtype=torch.float16
).to(<span class="hljs-string">&quot;cuda&quot;</span>)  <span class="hljs-comment"># or &quot;mps&quot;, &quot;xpu&quot;, &quot;cpu&quot;</span>

<span class="hljs-comment"># torch.compile</span>
pipeline.transformer.to(memory_format=torch.channels_last)
pipeline.transformer = torch.<span class="hljs-built_in">compile</span>(
    pipeline.transformer, mode=<span class="hljs-string">&quot;max-autotune&quot;</span>, fullgraph=<span class="hljs-literal">True</span>
)

prompt = <span class="hljs-string">&quot;&quot;&quot;
A detailed wooden toy ship with intricately carved masts and sails is seen gliding smoothly over a plush, blue carpet that mimics the waves of the sea. 
The ship&#x27;s hull is painted a rich brown, with tiny windows. The carpet, soft and textured, provides a perfect backdrop, resembling an oceanic expanse. 
Surrounding the ship are various other toys and children&#x27;s items, hinting at a playful environment. The scene captures the innocence and imagination of childhood, 
with the toy ship&#x27;s journey symbolizing endless adventures in a whimsical, indoor setting.
&quot;&quot;&quot;</span>

video = pipeline(
  prompt=prompt,
  guidance_scale=<span class="hljs-number">6</span>,
  num_inference_steps=<span class="hljs-number">50</span>
).frames[<span class="hljs-number">0</span>]
export_to_video(video, <span class="hljs-string">&quot;output.mp4&quot;</span>, fps=<span class="hljs-number">8</span>)<!----></pre></div><!----> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/diffusers/blob/main/docs/source/en/using-diffusers/text-img2vid.md" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg><!----> <span><span class="underline">Update</span> on GitHub</span></a><!----> <p></p><!--]--><!----><!--]--><!--]--><!--]--> <!--[-1--><!--]--><!--]-->
			
			<script>
				{
					__sveltekit_1qsdbwf = {
						base: "/docs/diffusers/main/en",
						assets: "/docs/diffusers/main/en"
					};
					const element = document.currentScript.parentElement;
					Promise.all([
						import("/docs/diffusers/main/en/_app/immutable/entry/start.0S0_iEoG.js"),
						import("/docs/diffusers/main/en/_app/immutable/entry/app.CPdOxCbw.js")
					]).then(([kit, app]) => {
						kit.start(app, element, {
							node_ids: [0, 407],
							data: [null,null],
							form: null,
							error: null
						});
					});
				}
			</script>
		
Xet Storage Details

Size:: 58.8 kB
Xet hash:: 9eb78d088bd5966f16513975d8e98dccd4cce1a3c31cec8390b3e5671e9c1a70
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.