Buckets:

hf-doc-build
/

doc

Files
xet
hf-doc-build/doc / diffusers /v0.43.0 /zh /optimization /speed-memory-optims.html
mishig
2 days ago
download
raw
27.5 kB
<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;编译和卸载量化模型&quot;,&quot;local&quot;:&quot;compiling-and-offloading-quantized-models&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;量化与 torch.compile&quot;,&quot;local&quot;:&quot;quantization-and-torchcompile&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;量化、torch.compile 和卸载&quot;,&quot;local&quot;:&quot;quantization-torchcompile-and-offloading&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}"/>
		<link href="/docs/diffusers/main/en/_app/immutable/entry/start.0S0_iEoG.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/I9eooN77.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/Bupkjr9f.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/entry/app.CPdOxCbw.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/CyvZ3FS5.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/l0QnnOED.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/DnVIJfC5.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/DsnmJJEf.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/nodes/0.DteTvmQZ.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/CoSrdQax.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/nodes/346.DtVZ4Ycr.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/DSPDl1gz.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/D5eijdnr.js" rel="modulepreload">
		<!--qpa0fj--><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;Compiling and offloading quantized models&quot;,&quot;local&quot;:&quot;compiling-and-offloading-quantized-models&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Quantization and torch.compile&quot;,&quot;local&quot;:&quot;quantization-and-torchcompile&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Quantization, torch.compile, and offloading&quot;,&quot;local&quot;:&quot;quantization-torchcompile-and-offloading&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}"/><!---->
		<link href="/docs/diffusers/main/en/_app/immutable/assets/0.tn0RQdqM.css" rel="modulepreload"> <!--[--><!--[0--><!--[--><!--[0--><!--[--><p></p> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: inline-end; margin-inline-start: 10px; display: inline-flex; position: relative; z-index: 10;"><div dir="ltr" class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 h-7 max-sm:h-7 px-2 max-sm:px-1.5 text-sm font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0 hover:text-gray-800 dark:hover:text-gray-200"><svg class="sm:size-3.5 size-3" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----></span> <span dir="auto">Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-7 max-sm:h-7 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible sm:size-3.5 size-3 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg><!----></button></div> <!--[-1--><!--]--></div><!----> <!--[0--><h1 class="relative group"><a id="compiling-and-offloading-quantized-models" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#compiling-and-offloading-quantized-models"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->编译和卸载量化模型<!----></span></h1><!--]--><!----> <p>优化模型通常需要在<a href="./fp16">推理速度</a>和<a href="./memory">内存占用</a>之间进行权衡。例如，<a href="./cache">缓存</a>虽然可以提升推理速度，但由于需要存储中间注意力层的输出，也会增加内存消耗。一种更为均衡的优化策略是将模型量化、<a href="./fp16#torchcompile">torch.compile</a>以及多种<a href="./memory#offloading">卸载方法</a>相结合。</p> <blockquote class="tip"><p>请参阅 <a href="./fp16#torchcompile">torch.compile</a> 指南，以了解有关编译的更多信息以及在此处如何应用它们。例如，区域编译可以在不牺牲任何加速效果的情况下显著缩短编译时间。</p></blockquote> <p>在图像生成中，将量化与<a href="./memory#model-offloading">模型卸载</a>相结合，通常能在质量、速度和内存之间取得最佳平衡。组卸载在图像生成中效果不佳，因为如果计算内核完成得更快，通常无法<em>完全</em>重叠数据传输。这会导致 CPU 和 GPU 之间产生一定的通信开销。</p> <p>对于视频生成，结合量化和 <a href="./memory#group-offloading">group-offloading</a> 通常效果更好，因为视频模型更受计算限制。</p> <p>下表提供了 Flux 优化策略组合及其对延迟和内存使用影响的比较。</p> <table><thead><tr><th>组合</th><th>延迟（秒）</th><th>内存使用量（GB）</th></tr></thead><tbody><tr><td>量化</td><td>32.602</td><td>14.9453</td></tr><tr><td>量化，torch.compile</td><td>25.847</td><td>14.9448</td></tr><tr><td>量化、torch.compile、模型 CPU 卸载</td><td>32.312</td><td>12.2369</td></tr></tbody></table> <small>These results are benchmarked on Flux with a RTX 4090. The transformer and text_encoder components are quantized. Refer to the <a href="https://gist.github.com/sayakpaul/0db9d8eeeb3d2a0e5ed7cf0d9ca19b7d">benchmarking script</a> if you're interested in evaluating your own model.</small> <p>本指南将演示如何使用 <a href="../quantization/bitsandbytes#torchcompile">bitsandbytes</a> 编译并卸载量化模型。请确保你使用的是 <a href="https://pytorch.org/get-started/locally/" rel="nofollow">PyTorch nightly</a> 以及 bitsandbytes 的最新版本。</p> <p>虽然本示例使用了 bitsandbytes，但其他量化后端（例如 <a href="../quantization/torchao">TorchAO</a>）也支持这些功能。</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> 已复制</div><!----></button><!----></div> <pre class="language-bash "><!---->pip install -U bitsandbytes<!----></pre></div><!----> <!--[1--><h2 class="relative group"><a id="quantization-and-torchcompile" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#quantization-and-torchcompile"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->量化与 torch.compile<!----></span></h2><!--]--><!----> <p>首先对模型进行<a href="../quantization/overview">量化</a>以降低存储所需的内存，然后对其进行<a href="./fp16#torchcompile">编译</a>以加速推理。</p> <p>配置 <a href="https://docs.pytorch.org/docs/stable/torch.compiler_dynamo_overview.html" rel="nofollow">Dynamo</a> 的 <code>capture_dynamic_output_shape_ops = True</code>，以在编译 bitsandbytes 模型时处理动态输出。</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> 已复制</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> DiffusionPipeline
<span class="hljs-keyword">from</span> diffusers.quantizers <span class="hljs-keyword">import</span> PipelineQuantizationConfig

torch._dynamo.config.capture_dynamic_output_shape_ops = <span class="hljs-literal">True</span>

<span class="hljs-comment"># quantize</span>
pipeline_quant_config = PipelineQuantizationConfig(
    quant_backend=<span class="hljs-string">&quot;bitsandbytes_4bit&quot;</span>,
    quant_kwargs={<span class="hljs-string">&quot;load_in_4bit&quot;</span>: <span class="hljs-literal">True</span>, <span class="hljs-string">&quot;bnb_4bit_quant_type&quot;</span>: <span class="hljs-string">&quot;nf4&quot;</span>, <span class="hljs-string">&quot;bnb_4bit_compute_dtype&quot;</span>: torch.bfloat16},
    components_to_quantize=[<span class="hljs-string">&quot;transformer&quot;</span>, <span class="hljs-string">&quot;text_encoder_2&quot;</span>],
)
pipeline = DiffusionPipeline.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    quantization_config=pipeline_quant_config,
    dtype=torch.bfloat16,
).to(<span class="hljs-string">&quot;cuda&quot;</span>)  <span class="hljs-comment"># or &quot;mps&quot;, &quot;xpu&quot;, &quot;cpu&quot;</span>

<span class="hljs-comment"># compile</span>
pipeline.transformer.to(memory_format=torch.channels_last)
pipeline.transformer.<span class="hljs-built_in">compile</span>(mode=<span class="hljs-string">&quot;max-autotune&quot;</span>, fullgraph=<span class="hljs-literal">True</span>)
pipeline(<span class="hljs-string">&quot;&quot;&quot;
    cinematic film still of a cat sipping a margarita in a pool in Palm Springs, California
    highly detailed, high budget hollywood movie, cinemascope, moody, epic, gorgeous, film grain
&quot;&quot;&quot;</span>
).images[<span class="hljs-number">0</span>]<!----></pre></div><!----> <!--[1--><h2 class="relative group"><a id="quantization-torchcompile-and-offloading" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#quantization-torchcompile-and-offloading"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->量化、torch.compile 和卸载<!----></span></h2><!--]--><!----> <p>除了量化和 torch.compile，如果还需要进一步降低内存占用，可以尝试使用 offloading。Offloading 会根据计算需求，将各个层或模型组件在 CPU 和 GPU 之间进行移动。</p> <p>在卸载过程中配置 <a href="https://docs.pytorch.org/docs/stable/torch.compiler_dynamo_overview.html" rel="nofollow">Dynamo</a> 的 <code>cache_size_limit</code> 以避免过度重新编译，并设置 <code>capture_dynamic_output_shape_ops = True</code> 以在处理 bitsandbytes 模型的编译时应对动态输出。</p> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"><!--[--><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd border-gray-800 bg-black dark:bg-gray-700 text-white">模型 CPU 卸载</div><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd text-gray-500 cursor-pointer opacity-90 hover:text-gray-700 dark:hover:text-gray-200 hover:shadow-sm">组卸载</div><!--]--></div> <div class="language-select"><div><p><a href="./memory#model-offloading">模型 CPU 卸载</a>会在需要计算时，将单个流水线组件（例如 transformer 模型）移至 GPU。否则，该组件会被卸载到 CPU。</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> 已复制</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> DiffusionPipeline
<span class="hljs-keyword">from</span> diffusers.quantizers <span class="hljs-keyword">import</span> PipelineQuantizationConfig

torch._dynamo.config.cache_size_limit = <span class="hljs-number">1000</span>
torch._dynamo.config.capture_dynamic_output_shape_ops = <span class="hljs-literal">True</span>

<span class="hljs-comment"># quantize</span>
pipeline_quant_config = PipelineQuantizationConfig(
    quant_backend=<span class="hljs-string">&quot;bitsandbytes_4bit&quot;</span>,
    quant_kwargs={<span class="hljs-string">&quot;load_in_4bit&quot;</span>: <span class="hljs-literal">True</span>, <span class="hljs-string">&quot;bnb_4bit_quant_type&quot;</span>: <span class="hljs-string">&quot;nf4&quot;</span>, <span class="hljs-string">&quot;bnb_4bit_compute_dtype&quot;</span>: torch.bfloat16},
    components_to_quantize=[<span class="hljs-string">&quot;transformer&quot;</span>, <span class="hljs-string">&quot;text_encoder_2&quot;</span>],
)
pipeline = DiffusionPipeline.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    quantization_config=pipeline_quant_config,
    dtype=torch.bfloat16,
).to(<span class="hljs-string">&quot;cuda&quot;</span>)  <span class="hljs-comment"># or &quot;mps&quot;, &quot;xpu&quot;, &quot;cpu&quot;</span>

<span class="hljs-comment"># model CPU offloading</span>
pipeline.enable_model_cpu_offload()

<span class="hljs-comment"># compile</span>
pipeline.transformer.<span class="hljs-built_in">compile</span>()
pipeline(
    <span class="hljs-string">&quot;cinematic film still of a cat sipping a margarita in a pool in Palm Springs, California, highly detailed, high budget hollywood movie, cinemascope, moody, epic, gorgeous, film grain&quot;</span>
).images[<span class="hljs-number">0</span>]<!----></pre></div><!----><!----></div><!----> <div hidden=""><p><a href="./memory#group-offloading">Group offloading</a> 会将单个流水线组件（例如 transformer 模型）的内部层移至 GPU 进行计算，并在不需要时将其卸载。同时，它利用 <a href="./memory#cuda-stream">CUDA stream</a> 特性预取下一层以执行。</p> <p>通过重叠计算和数据传输，它比模型 CPU 卸载更快，同时还能节省内存。</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> 已复制</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-comment"># pip install ftfy</span>
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> AutoModel, DiffusionPipeline
<span class="hljs-keyword">from</span> diffusers.hooks <span class="hljs-keyword">import</span> apply_group_offloading
<span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video
<span class="hljs-keyword">from</span> diffusers.quantizers <span class="hljs-keyword">import</span> PipelineQuantizationConfig
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> UMT5EncoderModel

torch._dynamo.config.cache_size_limit = <span class="hljs-number">1000</span>
torch._dynamo.config.capture_dynamic_output_shape_ops = <span class="hljs-literal">True</span>

<span class="hljs-comment"># quantize</span>
pipeline_quant_config = PipelineQuantizationConfig(
    quant_backend=<span class="hljs-string">&quot;bitsandbytes_4bit&quot;</span>,
    quant_kwargs={<span class="hljs-string">&quot;load_in_4bit&quot;</span>: <span class="hljs-literal">True</span>, <span class="hljs-string">&quot;bnb_4bit_quant_type&quot;</span>: <span class="hljs-string">&quot;nf4&quot;</span>, <span class="hljs-string">&quot;bnb_4bit_compute_dtype&quot;</span>: torch.bfloat16},
    components_to_quantize=[<span class="hljs-string">&quot;transformer&quot;</span>, <span class="hljs-string">&quot;text_encoder&quot;</span>],
)

text_encoder = UMT5EncoderModel.from_pretrained(
    <span class="hljs-string">&quot;Wan-AI/Wan2.1-T2V-14B-Diffusers&quot;</span>, subfolder=<span class="hljs-string">&quot;text_encoder&quot;</span>, dtype=torch.bfloat16
)
pipeline = DiffusionPipeline.from_pretrained(
    <span class="hljs-string">&quot;Wan-AI/Wan2.1-T2V-14B-Diffusers&quot;</span>,
    quantization_config=pipeline_quant_config,
    dtype=torch.bfloat16,
).to(<span class="hljs-string">&quot;cuda&quot;</span>)  <span class="hljs-comment"># or &quot;mps&quot;, &quot;xpu&quot;, &quot;cpu&quot;</span>

<span class="hljs-comment"># group offloading</span>
onload_device = torch.device(<span class="hljs-string">&quot;cuda&quot;</span>)
offload_device = torch.device(<span class="hljs-string">&quot;cpu&quot;</span>)

pipeline.transformer.enable_group_offload(
    onload_device=onload_device,
    offload_device=offload_device,
    offload_type=<span class="hljs-string">&quot;leaf_level&quot;</span>,
    use_stream=<span class="hljs-literal">True</span>,
    non_blocking=<span class="hljs-literal">True</span>
)
pipeline.vae.enable_group_offload(
    onload_device=onload_device,
    offload_device=offload_device,
    offload_type=<span class="hljs-string">&quot;leaf_level&quot;</span>,
    use_stream=<span class="hljs-literal">True</span>,
    non_blocking=<span class="hljs-literal">True</span>
)
apply_group_offloading(
    pipeline.text_encoder,
    onload_device=onload_device,
    offload_type=<span class="hljs-string">&quot;leaf_level&quot;</span>,
    use_stream=<span class="hljs-literal">True</span>,
    non_blocking=<span class="hljs-literal">True</span>
)

<span class="hljs-comment"># compile</span>
pipeline.transformer.<span class="hljs-built_in">compile</span>()

prompt = <span class="hljs-string">&quot;&quot;&quot;
The camera rushes from far to near in a low-angle shot, 
revealing a white ferret on a log. It plays, leaps into the water, and emerges, as the camera zooms in 
for a close-up. Water splashes berry bushes nearby, while moss, snow, and leaves blanket the ground. 
Birch trees and a light blue sky frame the scene, with ferns in the foreground. Side lighting casts dynamic 
shadows and warm highlights. Medium composition, front view, low angle, with depth of field.
&quot;&quot;&quot;</span>
negative_prompt = <span class="hljs-string">&quot;&quot;&quot;
Bright tones, overexposed, static, blurred details, subtitles, style, works, paintings, images, static, overall gray, worst quality, 
low quality, JPEG compression residue, ugly, incomplete, extra fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured, 
misshapen limbs, fused fingers, still picture, messy background, three legs, many people in the background, walking backwards
&quot;&quot;&quot;</span>

output = pipeline(
    prompt=prompt,
    negative_prompt=negative_prompt,
    num_frames=<span class="hljs-number">81</span>,
    guidance_scale=<span class="hljs-number">5.0</span>,
).frames[<span class="hljs-number">0</span>]
export_to_video(output, <span class="hljs-string">&quot;output.mp4&quot;</span>, fps=<span class="hljs-number">16</span>)<!----></pre></div><!----><!----></div><!----><!----></div><!----> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/diffusers/blob/main/docs/source/en/optimization/speed-memory-optims.md" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg><!----> <span><span class="underline">Update</span> on GitHub</span></a><!----> <p></p><!--]--><!----><!--]--><!--]--><!--]--> <!--[-1--><!--]--><!--]-->
			
			<script>
				{
					__sveltekit_1qsdbwf = {
						base: "/docs/diffusers/main/en",
						assets: "/docs/diffusers/main/en"
					};
					const element = document.currentScript.parentElement;
					Promise.all([
						import("/docs/diffusers/main/en/_app/immutable/entry/start.0S0_iEoG.js"),
						import("/docs/diffusers/main/en/_app/immutable/entry/app.CPdOxCbw.js")
					]).then(([kit, app]) => {
						kit.start(app, element, {
							node_ids: [0, 346],
							data: [null,null],
							form: null,
							error: null
						});
					});
				}
			</script>
		
Xet Storage Details

Size:: 27.5 kB
Xet hash:: b3825623f15a7b770e9bc7c9772db758d45b86b7181b33942a80c6b402d0ed31
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.