Hugging Face's logo

Buckets:
hf-doc-build
/
doc

Files
xet
hf-doc-build/doc / diffusers /v0.43.0 /id /quantization /bfl.html
mishig's picture
mishig
download
raw
14.1 kB
<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;BFL (fp8r / nvfp4)&quot;,&quot;local&quot;:&quot;bfl-fp8r--nvfp4&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Kernels&quot;,&quot;local&quot;:&quot;kernels&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}"/>
<link href="/docs/diffusers/main/en/_app/immutable/entry/start.0S0_iEoG.js" rel="modulepreload">
<link href="/docs/diffusers/main/en/_app/immutable/chunks/I9eooN77.js" rel="modulepreload">
<link href="/docs/diffusers/main/en/_app/immutable/chunks/Bupkjr9f.js" rel="modulepreload">
<link href="/docs/diffusers/main/en/_app/immutable/entry/app.CPdOxCbw.js" rel="modulepreload">
<link href="/docs/diffusers/main/en/_app/immutable/chunks/CyvZ3FS5.js" rel="modulepreload">
<link href="/docs/diffusers/main/en/_app/immutable/chunks/l0QnnOED.js" rel="modulepreload">
<link href="/docs/diffusers/main/en/_app/immutable/chunks/DnVIJfC5.js" rel="modulepreload">
<link href="/docs/diffusers/main/en/_app/immutable/chunks/DsnmJJEf.js" rel="modulepreload">
<link href="/docs/diffusers/main/en/_app/immutable/nodes/0.DteTvmQZ.js" rel="modulepreload">
<link href="/docs/diffusers/main/en/_app/immutable/chunks/CoSrdQax.js" rel="modulepreload">
<link href="/docs/diffusers/main/en/_app/immutable/nodes/351.Cu90JRCw.js" rel="modulepreload">
<link href="/docs/diffusers/main/en/_app/immutable/chunks/DSPDl1gz.js" rel="modulepreload">
<!--7gq4hb--><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;BFL (fp8r / nvfp4)&quot;,&quot;local&quot;:&quot;bfl-fp8r--nvfp4&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Kernels&quot;,&quot;local&quot;:&quot;kernels&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}"/><!---->
<link href="/docs/diffusers/main/en/_app/immutable/assets/0.tn0RQdqM.css" rel="modulepreload"> <!--[--><!--[0--><!--[--><!--[0--><!--[--><p></p> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: inline-end; margin-inline-start: 10px; display: inline-flex; position: relative; z-index: 10;"><div dir="ltr" class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 h-7 max-sm:h-7 px-2 max-sm:px-1.5 text-sm font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0 hover:text-gray-800 dark:hover:text-gray-200"><svg class="sm:size-3.5 size-3" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----></span> <span dir="auto">Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-7 max-sm:h-7 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible sm:size-3.5 size-3 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg><!----></button></div> <!--[-1--><!--]--></div><!----> <!--[0--><h1 class="relative group"><a id="bfl-fp8r--nvfp4" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#bfl-fp8r--nvfp4"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->BFL (fp8r / nvfp4)<!----></span></h1><!--]--><!----> <p><a href="https://huggingface.co/black-forest-labs" rel="nofollow">Black Forest Labs</a> menyediakan varian terkuantisasi dari <a href="../api/pipelines/flux3">Flux3</a> transformers-nya dalam dua skema, disimpan sebagai checkpoint file tunggal di samping bobot bf16 aslinya:</p> <ul><li><code>fp8r</code>: FP8 rowwise. Setiap linear terkuantisasi menyimpan <code>weight</code> E4M3 dan <code>weight_scale</code> fp32 per baris. Aktivasi dkuantisasi ke E4M3 per baris saat runtime.</li> <li><code>nvfp4</code>: NVFP4 dalam tata letak <a href="https://github.com/NVIDIA/Model-Optimizer" rel="nofollow">ModelOpt</a>. Setiap linear terkuantisasi menyimpan <code>weight</code> E2M1 yang dikemas (dua nilai per byte), skala blok E4M3 sebesar 16 (<code>weight_scale</code>), <code>weight_scale_2</code> global fp32, dan skala aktivasi fp32 yang dikalibrasi <code>input_scale</code>.</li></ul> <p>Diffusers memuat keduanya melalui <code>~FromSingleFileMixin.from_single_file</code> dengan <a href="/docs/diffusers/v0.43.0/id/api/quantization#diffusers.BFLQuantizationConfig">BFLQuantizationConfig</a>. Skema dibaca dari checkpoint, sehingga config yang sama berfungsi untuk kedua file. Hanya pemuatan checkpoint prequantized yang didukung; pemuatan tingkat pipeline tidak didukung.</p> <p>Contoh di bawah ini memuat checkpoint <code>fp8r</code> dari <a href="https://huggingface.co/black-forest-labs/flux-3-image-klein-base" rel="nofollow">FLUX 3 Image Klein Base</a>, yang berada di <code>variants/fp8r/model.safetensors</code> dalam repositori model. Sampaikan <code>config</code> agar konfigurasi model dibaca dari folder Diffusers di repositori tersebut, lalu muat komponen lainnya berdasarkan nama (lihat <a href="../api/pipelines/flux3#single-file-loading">Pemuatan file tunggal</a>).</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Disalin</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> BFLQuantizationConfig, Flux3Transformer2DModel, ModularPipeline
ckpt_path = <span class="hljs-string">&quot;https://huggingface.co/black-forest-labs/flux-3-image-klein-base/blob/main/variants/fp8r/model.safetensors&quot;</span>
transformer = Flux3Transformer2DModel.from_single_file(
ckpt_path,
config=<span class="hljs-string">&quot;black-forest-labs/flux-3-image-klein-base&quot;</span>,
subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
quantization_config=BFLQuantizationConfig(compute_dtype=torch.bfloat16),
dtype=torch.bfloat16,
)
pipe = ModularPipeline.from_pretrained(<span class="hljs-string">&quot;black-forest-labs/flux-3-image-klein-base&quot;</span>)
pipe.load_components(dtype=torch.bfloat16)
pipe.update_components(transformer=transformer)
pipe.to(<span class="hljs-string">&quot;cuda&quot;</span>)
prompt = <span class="hljs-string">&quot;A wide-angle wildlife photograph of a red fox standing alert in a snow-covered forest clearing at golden hour.&quot;</span>
image = pipe(prompt=prompt, generator=torch.Generator().manual_seed(<span class="hljs-number">0</span>), output=<span class="hljs-string">&quot;images&quot;</span>)[<span class="hljs-number">0</span>]
image.save(<span class="hljs-string">&quot;flux3-klein-base-fp8r.png&quot;</span>)<!----></pre></div><!----> <p>Ganti <code>variants/fp8r</code> dengan <code>variants/nvfp4</code> untuk memuat varian <code>nvfp4</code>. <a href="https://huggingface.co/black-forest-labs/flux-3-image-klein" rel="nofollow">FLUX 3 Image Klein</a> yang telah di-distilasi langkahnya menyertakan dua varian yang sama; muat keduanya dengan id repositorinya sebagai <code>config</code> dan pipeline.</p> <!--[1--><h2 class="relative group"><a id="kernels" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#kernels"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Kernels<!----></span></h2><!--]--><!----> <p>Bobot terkuantisasi tetap berada dalam dtype penyimpanan presisi rendah. GEMM mana yang dijalankan bergantung pada perangkat:</p> <table><thead><tr><th>Skema</th><th>Kernel terpadu</th><th>Persyaratan</th></tr></thead><tbody><tr><td><code>fp8r</code></td><td><code>torch._scaled_mm</code> dengan skala baris</td><td>GPU CUDA dengan kemampuan komputasi 8.9 atau lebih tinggi</td></tr><tr><td><code>nvfp4</code></td><td><code>flashinfer.mm_fp4</code> (CUTLASS) dengan <code>flashinfer.nvfp4_quantize</code> untuk aktivasi</td><td>GPU Blackwell (kemampuan komputasi 10.0 atau lebih tinggi), toolkit CUDA dengan <code>nvcc</code>, dan <code>flashinfer-python</code> dengan <code>flashinfer-cubin</code> pada versi yang sama</td></tr></tbody></table> <p>Install kedua paket FlashInfer pada versi yang sama, misalnya <code>pip install flashinfer-python==0.6.13 flashinfer-cubin==0.6.13</code>. Rilis <code>flashinfer-cubin</code> bisa tertinggal di belakang <code>flashinfer-python</code>, dan FlashInfer akan menolak diimpor jika versinya berbeda.</p> <p>FlashInfer mengompilasi kernel NVFP4-nya pada penggunaan pertama dengan toolkit CUDA dari lingkungan dan memilih arsitektur target dari GPU yang terlihat (atau dari <code>FLASHINFER_CUDA_ARCH_LIST</code> jika disetel). GPU Blackwell untuk konsumen dan workstation (compute capability 12.0) memerlukan toolkit CUDA versi 12.9 atau lebih baru. Dengan toolkit yang lebih lama, FlashInfer tidak menemukan arsitektur yang memenuhi syarat dan panggilan NVFP4 pertama akan gagal dengan <code>FlashInfer requires GPUs with sm75 or higher</code>.</p> <p>Pada perangkat lain, bobot didekuantisasi ke <code>compute_dtype</code> pada setiap forward pass dan layer berjalan sebagai linear reguler. Dalam kasus tersebut, aktivasi tidak dikuantisasi, sehingga output sedikit berbeda dari kernel yang difusi.</p> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/diffusers/blob/main/docs/source/en/quantization/bfl.md" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg><!----> <span><span class="underline">Update</span> on GitHub</span></a><!----> <p></p><!--]--><!----><!--]--><!--]--><!--]--> <!--[-1--><!--]--><!--]-->
<script>
{
__sveltekit_1qsdbwf = {
base: "/docs/diffusers/main/en",
assets: "/docs/diffusers/main/en"
};
const element = document.currentScript.parentElement;
Promise.all([
import("/docs/diffusers/main/en/_app/immutable/entry/start.0S0_iEoG.js"),
import("/docs/diffusers/main/en/_app/immutable/entry/app.CPdOxCbw.js")
]).then(([kit, app]) => {
kit.start(app, element, {
node_ids: [0, 351],
data: [null,null],
form: null,
error: null
});
});
}
</script>

Xet Storage Details

Size:
14.1 kB
·
Xet hash:
c6c01a23511e2e04a1c39e1b355d0fd3f83ee7962b3b952aab98879deae41226

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.