Hugging Face's logo

Buckets:
hf-doc-build
/
doc

Files
xet
hf-doc-build/doc / diffusers /v0.43.0 /pt /quantization /torchao.html
mishig's picture
mishig
download
raw
33.8 kB
<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;torchao&quot;,&quot;local&quot;:&quot;torchao&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;torch.compile&quot;,&quot;local&quot;:&quot;torchcompile&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Tipos de quantização suportados&quot;,&quot;local&quot;:&quot;supported-quantization-types&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Serialização e desserialização de modelos quantizados&quot;,&quot;local&quot;:&quot;serializing-and-deserializing-quantized-models&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Recursos&quot;,&quot;local&quot;:&quot;resources&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}"/>
<link href="/docs/diffusers/main/en/_app/immutable/entry/start.0S0_iEoG.js" rel="modulepreload">
<link href="/docs/diffusers/main/en/_app/immutable/chunks/I9eooN77.js" rel="modulepreload">
<link href="/docs/diffusers/main/en/_app/immutable/chunks/Bupkjr9f.js" rel="modulepreload">
<link href="/docs/diffusers/main/en/_app/immutable/entry/app.CPdOxCbw.js" rel="modulepreload">
<link href="/docs/diffusers/main/en/_app/immutable/chunks/CyvZ3FS5.js" rel="modulepreload">
<link href="/docs/diffusers/main/en/_app/immutable/chunks/l0QnnOED.js" rel="modulepreload">
<link href="/docs/diffusers/main/en/_app/immutable/chunks/DnVIJfC5.js" rel="modulepreload">
<link href="/docs/diffusers/main/en/_app/immutable/chunks/DsnmJJEf.js" rel="modulepreload">
<link href="/docs/diffusers/main/en/_app/immutable/nodes/0.DteTvmQZ.js" rel="modulepreload">
<link href="/docs/diffusers/main/en/_app/immutable/chunks/CoSrdQax.js" rel="modulepreload">
<link href="/docs/diffusers/main/en/_app/immutable/nodes/359.DjT-ulWG.js" rel="modulepreload">
<link href="/docs/diffusers/main/en/_app/immutable/chunks/DSPDl1gz.js" rel="modulepreload">
<!--1q5ej5h--><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;torchao&quot;,&quot;local&quot;:&quot;torchao&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;torch.compile&quot;,&quot;local&quot;:&quot;torchcompile&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Supported quantization types&quot;,&quot;local&quot;:&quot;supported-quantization-types&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Serializing and Deserializing quantized models&quot;,&quot;local&quot;:&quot;serializing-and-deserializing-quantized-models&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Resources&quot;,&quot;local&quot;:&quot;resources&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}"/><!---->
<link href="/docs/diffusers/main/en/_app/immutable/assets/0.tn0RQdqM.css" rel="modulepreload"> <!--[--><!--[0--><!--[--><!--[0--><!--[--><p></p> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: inline-end; margin-inline-start: 10px; display: inline-flex; position: relative; z-index: 10;"><div dir="ltr" class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 h-7 max-sm:h-7 px-2 max-sm:px-1.5 text-sm font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0 hover:text-gray-800 dark:hover:text-gray-200"><svg class="sm:size-3.5 size-3" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----></span> <span dir="auto">Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-7 max-sm:h-7 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible sm:size-3.5 size-3 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg><!----></button></div> <!--[-1--><!--]--></div><!----> <!--[0--><h1 class="relative group"><a id="torchao" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#torchao"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->torchao<!----></span></h1><!--]--><!----> <p><a href="https://github.com/pytorch/ao" rel="nofollow">torchao</a> fornece dtypes de alto desempenho e otimizações baseadas em quantização e esparsidade para inferência e treinamento de modelos PyTorch. É suportado para qualquer modelo em qualquer modalidade, desde que suporte carregamento com <a href="https://hf.co/docs/accelerate/index" rel="nofollow">Accelerate</a> e contenha camadas <code>torch.nn.Linear</code>.</p> <p>Certifique-se de que o Pytorch 2.5+ e o torchao estão instalados com o comando abaixo.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copiado</div><!----></button><!----></div> <pre class="language-bash "><!---->uv pip install -U torch torchao<!----></pre></div><!----> <p>Cada dtype de quantização está disponível como uma instância separada de uma classe <a href="https://docs.pytorch.org/ao/main/api_reference/generated/torchao.core.config.AOBaseConfig.html" rel="nofollow">AOBaseConfig</a>. Isso fornece opções de configuração mais flexíveis ao expor mais argumentos disponíveis.</p> <p>Passe o <code>AOBaseConfig</code> de um dtype de quantização, como <a href="https://docs.pytorch.org/ao/main/api_reference/generated/torchao.quantization.Int4WeightOnlyConfig.html" rel="nofollow">Int4WeightOnlyConfig</a>, para <a href="/docs/diffusers/v0.43.0/pt/api/quantization#diffusers.TorchAoConfig">TorchAoConfig</a> em <a href="/docs/diffusers/v0.43.0/pt/api/models/overview#diffusers.ModelMixin.from_pretrained">from_pretrained()</a>.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copiado</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> DiffusionPipeline, PipelineQuantizationConfig, TorchAoConfig
<span class="hljs-keyword">from</span> torchao.quantization <span class="hljs-keyword">import</span> Int8WeightOnlyConfig
pipeline_quant_config = PipelineQuantizationConfig(
quant_mapping={<span class="hljs-string">&quot;transformer&quot;</span>: TorchAoConfig(Int8WeightOnlyConfig(group_size=<span class="hljs-number">128</span>, version=<span class="hljs-number">2</span>))}
)
pipeline = DiffusionPipeline.from_pretrained(
<span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
quantization_config=pipeline_quant_config,
dtype=torch.bfloat16,
device_map=<span class="hljs-string">&quot;cuda&quot;</span> <span class="hljs-comment"># or &quot;mps&quot;, &quot;xpu&quot;, &quot;cpu&quot;</span>
)<!----></pre></div><!----> <p><code>device_map="cuda"</code> quantiza cada camada na GPU durante o carregamento. Isso é rápido, mas requer temporariamente memória de GPU adicional para os pesos originais e quantizados. Se o modelo já usa a maior parte da memória da sua GPU, o carregamento pode falhar com um erro de falta de memória. Nesse caso, remova <code>device_map</code> e mova o pipeline para a GPU após o carregamento:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copiado</div><!----></button><!----></div> <pre class="language-py "><!---->pipeline = DiffusionPipeline.from_pretrained(
<span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
quantization_config=pipeline_quant_config,
torch_dtype=torch.bfloat16,
)
pipeline.to(<span class="hljs-string">&quot;cuda&quot;</span>) <span class="hljs-comment"># or &quot;mps&quot;, &quot;xpu&quot;, &quot;cpu&quot;</span><!----></pre></div><!----> <p>Sem <code>device_map</code>, o Diffusers quantiza as camadas na CPU. Isso é mais lento, mas evita o pico temporário de memória de GPU durante a quantização. Para reduzir ainda mais o uso de memória de GPU, use <a href="/docs/diffusers/v0.43.0/pt/api/pipelines/overview#diffusers.DiffusionPipeline.enable_model_cpu_offload">enable_model_cpu_offload()</a> no lugar. Você também pode quantizar componentes adicionais, como o text encoder.</p> <!--[1--><h2 class="relative group"><a id="torchcompile" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#torchcompile"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->torch.compile<!----></span></h2><!--]--><!----> <p>O torchao oferece suporte a <a href="../optimization/fp16#torchcompile">torch.compile</a>, que pode acelerar a inferência com uma única linha de código.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copiado</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> DiffusionPipeline, PipelineQuantizationConfig, TorchAoConfig
<span class="hljs-keyword">from</span> torchao.quantization <span class="hljs-keyword">import</span> Int4WeightOnlyConfig
pipeline_quant_config = PipelineQuantizationConfig(
quant_mapping={<span class="hljs-string">&quot;transformer&quot;</span>: TorchAoConfig(Int4WeightOnlyConfig(group_size=<span class="hljs-number">128</span>))}
)
pipeline = DiffusionPipeline.from_pretrained(
<span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
quantization_config=pipeline_quant_config,
dtype=torch.bfloat16,
device_map=<span class="hljs-string">&quot;cuda&quot;</span> <span class="hljs-comment"># or &quot;mps&quot;, &quot;xpu&quot;, &quot;cpu&quot;</span>
)
pipeline.transformer.<span class="hljs-built_in">compile</span>(transformer, mode=<span class="hljs-string">&quot;max-autotune&quot;</span>, fullgraph=<span class="hljs-literal">True</span>)<!----></pre></div><!----> <p>Consulte esta <a href="https://github.com/huggingface/diffusers/pull/10009#issue-2688781450" rel="nofollow">tabela</a> para benchmarks de velocidade de inferência e uso de memória com Flux e CogVideoX. Mais benchmarks em diversos hardwares também estão disponíveis no <a href="https://github.com/pytorch/ao/tree/main/torchao/quantization#benchmarks" rel="nofollow">repositório</a> do torchao.</p> <blockquote class="tip"><p>Os esquemas de quantização pós-treino em FP8 no torchao são eficazes para GPUs com capacidade de computação de pelo menos 8.9 (RTX-4090, Hopper, etc.). O FP8 frequentemente oferece o melhor equilíbrio entre velocidade, memória e qualidade ao gerar imagens e vídeos. Recomendamos combinar FP8 e torch.compile se sua GPU for compatível.</p></blockquote> <!--[1--><h2 class="relative group"><a id="supported-quantization-types" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#supported-quantization-types"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Tipos de quantização suportados<!----></span></h2><!--]--><!----> <p>O torchao suporta quantização apenas de pesos e quantização de pesos e ativações dinâmicas para int8, float3-float8 e uint1-uint7.</p> <p>A quantização apenas de pesos armazena os pesos do modelo em um tipo de dados de baixa bit, mas executa o cálculo com um tipo de dados de maior precisão, como <code>bfloat16</code>. Isso reduz os requisitos de memória dos pesos do modelo, mas mantém os picos de memória para o cálculo de ativações.</p> <p>A quantização dinâmica de ativações armazena os pesos do modelo em um dtype de baixo bit, quantizando também as ativações em tempo real para economizar memória adicional. Isso reduz os requisitos de memória dos pesos do modelo, além de reduzir a sobrecarga de memória das computações de ativação. No entanto, isso pode implicar em um compromisso de qualidade em alguns casos, portanto, é recomendado testar diferentes modelos minuciosamente.</p> <p>Consulte a <a href="https://docs.pytorch.org/ao/stable/index.html" rel="nofollow">documentação oficial do torchao</a> para uma melhor compreensão dos métodos de quantização disponíveis. Uma lista exaustiva das opções de configuração está disponível <a href="https://docs.pytorch.org/ao/main/workflows/inference.html#inference-workflows" rel="nofollow">aqui</a>.</p> <p>Alguns exemplos de configurações populares de quantização são os seguintes:</p> <table><thead><tr><th><strong>Categoria</strong></th><th><strong>Classes de Configuração</strong></th></tr></thead><tbody><tr><td><strong>Quantização de inteiros</strong></td><td><a href="https://docs.pytorch.org/ao/stable/api_reference/generated/torchao.quantization.Int4WeightOnlyConfig.html" rel="nofollow"><code>Int4WeightOnlyConfig</code></a>, <a href="https://docs.pytorch.org/ao/stable/api_reference/generated/torchao.quantization.Int8WeightOnlyConfig.html" rel="nofollow"><code>Int8WeightOnlyConfig</code></a>, <a href="https://docs.pytorch.org/ao/stable/api_reference/generated/torchao.quantization.Int8DynamicActivationInt8WeightConfig.html" rel="nofollow"><code>Int8DynamicActivationInt8WeightConfig</code></a></td></tr><tr><td><strong>Quantização de ponto flutuante de 8 bits</strong></td><td><a href="https://docs.pytorch.org/ao/stable/api_reference/generated/torchao.quantization.Float8WeightOnlyConfig.html" rel="nofollow"><code>Float8WeightOnlyConfig</code></a>, <a href="https://docs.pytorch.org/ao/stable/api_reference/generated/torchao.quantization.Float8DynamicActivationFloat8WeightConfig.html" rel="nofollow"><code>Float8DynamicActivationFloat8WeightConfig</code></a></td></tr><tr><td><strong>Quantização de inteiros sem sinal</strong></td><td><a href="https://docs.pytorch.org/ao/stable/api_reference/generated/torchao.quantization.IntxWeightOnlyConfig.html" rel="nofollow"><code>IntxWeightOnlyConfig</code></a></td></tr></tbody></table> <!--[1--><h2 class="relative group"><a id="serializing-and-deserializing-quantized-models" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#serializing-and-deserializing-quantized-models"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Serialização e desserialização de modelos quantizados<!----></span></h2><!--]--><!----> <p>Para serializar um modelo quantizado em um dtype específico, primeiro carregue o modelo com o dtype de quantização desejado e, em seguida, salve-o usando o método <a href="/docs/diffusers/v0.43.0/pt/api/models/overview#diffusers.ModelMixin.save_pretrained">save_pretrained()</a>.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copiado</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> AutoModel, TorchAoConfig
<span class="hljs-keyword">from</span> torchao.quantization <span class="hljs-keyword">import</span> Int8WeightOnlyConfig
quantization_config = TorchAoConfig(Int8WeightOnlyConfig())
transformer = AutoModel.from_pretrained(
<span class="hljs-string">&quot;black-forest-labs/Flux.1-Dev&quot;</span>,
subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
quantization_config=quantization_config,
dtype=torch.bfloat16,
)
transformer.save_pretrained(<span class="hljs-string">&quot;/path/to/flux_int8wo&quot;</span>, safe_serialization=<span class="hljs-literal">False</span>)<!----></pre></div><!----> <p>Para carregar um modelo quantizado serializado, use o método <a href="/docs/diffusers/v0.43.0/pt/api/models/overview#diffusers.ModelMixin.from_pretrained">from_pretrained()</a>.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copiado</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> FluxPipeline, AutoModel
transformer = AutoModel.from_pretrained(<span class="hljs-string">&quot;/path/to/flux_int8wo&quot;</span>, dtype=torch.bfloat16, use_safetensors=<span class="hljs-literal">False</span>)
pipe = FluxPipeline.from_pretrained(<span class="hljs-string">&quot;black-forest-labs/Flux.1-Dev&quot;</span>, transformer=transformer, dtype=torch.bfloat16)
pipe.to(<span class="hljs-string">&quot;cuda&quot;</span>) <span class="hljs-comment"># or &quot;mps&quot;, &quot;xpu&quot;, &quot;cpu&quot;</span>
prompt = <span class="hljs-string">&quot;A cat holding a sign that says hello world&quot;</span>
image = pipe(prompt, num_inference_steps=<span class="hljs-number">30</span>, guidance_scale=<span class="hljs-number">7.0</span>).images[<span class="hljs-number">0</span>]
image.save(<span class="hljs-string">&quot;output.png&quot;</span>)<!----></pre></div><!----> <p>Se você estiver usando <code>torch&lt;=2.6.0</code>, alguns métodos de quantização, como <code>uint4</code> weight-only, não podem ser carregados diretamente e podem resultar em um <code>UnpicklingError</code> ao tentar carregar os modelos, mas funcionam como esperado ao salvá-los. Para contornar isso, é possível carregar o state dict manualmente no modelo. Observe, no entanto, que isso requer o uso de <code>weights_only=False</code> em <code>torch.load</code>, portanto, deve ser executado apenas se os pesos tiverem sido obtidos de uma fonte confiável.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copiado</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> accelerate <span class="hljs-keyword">import</span> init_empty_weights
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> FluxPipeline, AutoModel, TorchAoConfig
<span class="hljs-keyword">from</span> torchao.quantization <span class="hljs-keyword">import</span> IntxWeightOnlyConfig
<span class="hljs-comment"># Serialize the model</span>
transformer = AutoModel.from_pretrained(
<span class="hljs-string">&quot;black-forest-labs/Flux.1-Dev&quot;</span>,
subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
quantization_config=TorchAoConfig(IntxWeightOnlyConfig(dtype=torch.uint4)),
dtype=torch.bfloat16,
)
transformer.save_pretrained(<span class="hljs-string">&quot;/path/to/flux_uint4wo&quot;</span>, safe_serialization=<span class="hljs-literal">False</span>, max_shard_size=<span class="hljs-string">&quot;50GB&quot;</span>)
<span class="hljs-comment"># ...</span>
<span class="hljs-comment"># Load the model</span>
state_dict = torch.load(<span class="hljs-string">&quot;/path/to/flux_uint4wo/diffusion_pytorch_model.bin&quot;</span>, weights_only=<span class="hljs-literal">False</span>, map_location=<span class="hljs-string">&quot;cpu&quot;</span>)
<span class="hljs-keyword">with</span> init_empty_weights():
transformer = AutoModel.from_config(<span class="hljs-string">&quot;/path/to/flux_uint4wo/config.json&quot;</span>)
transformer.load_state_dict(state_dict, strict=<span class="hljs-literal">True</span>, assign=<span class="hljs-literal">True</span>)<!----></pre></div><!----> <blockquote class="tip"><p>A API <a href="/docs/diffusers/v0.43.0/pt/api/models/auto_model#diffusers.AutoModel">AutoModel</a> é suportada para PyTorch &gt;= 2.6, conforme mostrado nos exemplos abaixo.</p></blockquote> <!--[1--><h2 class="relative group"><a id="resources" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#resources"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Recursos<!----></span></h2><!--]--><!----> <ul><li><a href="https://docs.pytorch.org/ao/stable/index.html" rel="nofollow">API de Quantização do TorchAO</a></li> <li><a href="https://github.com/sayakpaul/diffusers-torchao" rel="nofollow">Exemplos de Diffusers-TorchAO</a></li></ul> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/diffusers/blob/main/docs/source/en/quantization/torchao.md" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg><!----> <span><span class="underline">Update</span> on GitHub</span></a><!----> <p></p><!--]--><!----><!--]--><!--]--><!--]--> <!--[-1--><!--]--><!--]-->
<script>
{
__sveltekit_1qsdbwf = {
base: "/docs/diffusers/main/en",
assets: "/docs/diffusers/main/en"
};
const element = document.currentScript.parentElement;
Promise.all([
import("/docs/diffusers/main/en/_app/immutable/entry/start.0S0_iEoG.js"),
import("/docs/diffusers/main/en/_app/immutable/entry/app.CPdOxCbw.js")
]).then(([kit, app]) => {
kit.start(app, element, {
node_ids: [0, 359],
data: [null,null],
form: null,
error: null
});
});
}
</script>

Xet Storage Details

Size:
33.8 kB
·
Xet hash:
2f5887d6899e25ea2de1ac5f4a97d44ea0987e99b6fa244ae5ac4b961b745eb9

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.