Hugging Face's logo

Buckets:
hf-doc-build
/
doc

Files
xet
hf-doc-build/doc / diffusers /v0.43.0 /id /optimization /context_parallelism.html
mishig's picture
mishig
download
raw
40.5 kB
<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;Paralelisme konteks&quot;,&quot;local&quot;:&quot;context-parallelism&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Ring Attention&quot;,&quot;local&quot;:&quot;ring-attention&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Perhatian Ulysses&quot;,&quot;local&quot;:&quot;ulysses-attention&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Perhatian Terpadu&quot;,&quot;local&quot;:&quot;unified-attention&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Ulysses Anything Attention&quot;,&quot;local&quot;:&quot;ulysses-anything-attention&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Ring Anything Attention&quot;,&quot;local&quot;:&quot;ring-anything-attention&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Aktifkan paralelisme konteks saat memuat&quot;,&quot;local&quot;:&quot;enable-context-parallelism-while-loading&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}"/>
<link href="/docs/diffusers/main/en/_app/immutable/entry/start.0S0_iEoG.js" rel="modulepreload">
<link href="/docs/diffusers/main/en/_app/immutable/chunks/I9eooN77.js" rel="modulepreload">
<link href="/docs/diffusers/main/en/_app/immutable/chunks/Bupkjr9f.js" rel="modulepreload">
<link href="/docs/diffusers/main/en/_app/immutable/entry/app.CPdOxCbw.js" rel="modulepreload">
<link href="/docs/diffusers/main/en/_app/immutable/chunks/CyvZ3FS5.js" rel="modulepreload">
<link href="/docs/diffusers/main/en/_app/immutable/chunks/l0QnnOED.js" rel="modulepreload">
<link href="/docs/diffusers/main/en/_app/immutable/chunks/DnVIJfC5.js" rel="modulepreload">
<link href="/docs/diffusers/main/en/_app/immutable/chunks/DsnmJJEf.js" rel="modulepreload">
<link href="/docs/diffusers/main/en/_app/immutable/nodes/0.DteTvmQZ.js" rel="modulepreload">
<link href="/docs/diffusers/main/en/_app/immutable/chunks/CoSrdQax.js" rel="modulepreload">
<link href="/docs/diffusers/main/en/_app/immutable/nodes/333.OuTuM9lH.js" rel="modulepreload">
<link href="/docs/diffusers/main/en/_app/immutable/chunks/DSPDl1gz.js" rel="modulepreload">
<!--ty2c99--><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;Context parallelism&quot;,&quot;local&quot;:&quot;context-parallelism&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Ring Attention&quot;,&quot;local&quot;:&quot;ring-attention&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Ulysses Attention&quot;,&quot;local&quot;:&quot;ulysses-attention&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Unified Attention&quot;,&quot;local&quot;:&quot;unified-attention&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Ulysses Anything Attention&quot;,&quot;local&quot;:&quot;ulysses-anything-attention&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Ring Anything Attention&quot;,&quot;local&quot;:&quot;ring-anything-attention&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Enable context parallelism while loading&quot;,&quot;local&quot;:&quot;enable-context-parallelism-while-loading&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}"/><!---->
<link href="/docs/diffusers/main/en/_app/immutable/assets/0.tn0RQdqM.css" rel="modulepreload"> <!--[--><!--[0--><!--[--><!--[0--><!--[--><p></p> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: inline-end; margin-inline-start: 10px; display: inline-flex; position: relative; z-index: 10;"><div dir="ltr" class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 h-7 max-sm:h-7 px-2 max-sm:px-1.5 text-sm font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0 hover:text-gray-800 dark:hover:text-gray-200"><svg class="sm:size-3.5 size-3" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----></span> <span dir="auto">Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-7 max-sm:h-7 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible sm:size-3.5 size-3 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg><!----></button></div> <!--[-1--><!--]--></div><!----> <!--[0--><h1 class="relative group"><a id="context-parallelism" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#context-parallelism"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Paralelisme konteks<!----></span></h1><!--]--><!----> <p><a href="https://huggingface.co/spaces/nanotron/ultrascale-playbook?section=context_parallelism" rel="nofollow">Context parallelism</a> membagi urutan input di antara GPU, dan setiap GPU memproses bagiannya sendiri. Hal ini menurunkan memori per GPU, yang paling membantu untuk urutan panjang seperti gambar resolusi tinggi dan video. Untuk membandingkannya dengan strategi lain, lihat <a href="../training/distributed_inference">Distributed inference</a>.</p> <p>Context parallelism berjalan di dalam attention backend, dan hanya beberapa backend yang mendukungnya. Jika backend Anda tidak mendukungnya, <code>enable_parallelism</code> akan memicu error yang mencantumkan backend yang kompatibel. Ganti backend dengan <a href="/docs/diffusers/v0.43.0/id/api/models/overview#diffusers.ModelMixin.set_attention_backend">set_attention_backend()</a>, dan lihat <a href="./attention_backends#available-backends">backend yang tersedia</a> untuk daftar lengkap.</p> <p>Konfigurasikan setiap varian di bawah dengan <a href="/docs/diffusers/v0.43.0/id/api/parallel#diffusers.ContextParallelConfig">ContextParallelConfig</a>.</p> <table><thead><tr><th>Varian</th><th>Konfigurasi</th><th>Persyaratan</th><th>Gunakan saat</th></tr></thead><tbody><tr><td><a href="#ring-attention">Ring Attention</a></td><td><code>ring_degree > 1</code></td><td>panjang sekuens harus dapat dibagi habis oleh <code>ring_degree</code></td><td>urutan panjang dan bandwidth interkoneksi terbatas</td></tr><tr><td><a href="#ulysses-attention">Ulysses Attention</a></td><td><code>ulysses_degree > 1</code></td><td>panjang sekuens dan jumlah kepala perhatian harus dapat dibagi habis oleh <code>ulysses_degree</code></td><td>GPU Anda memiliki interkoneksi yang cepat</td></tr><tr><td><a href="#unified-attention">Atensi Terpadu</a></td><td><code>ring_degree > 1</code> dan <code>ulysses_degree > 1</code></td><td>minimal 4 GPU</td><td>Anda memiliki lebih banyak perangkat daripada yang dapat digunakan Ulysses Attention secara mandiri</td></tr><tr><td><a href="#ulysses-anything-attention">Ulysses Anything Attention</a></td><td><code>ulysses_degree > 1</code>, <code>ulysses_anything=True</code></td><td><code>ring_degree</code> harus 1</td><td>panjang sekuens atau jumlah head tidak terbagi rata</td></tr><tr><td><a href="#ring-anything-attention">Ring Anything Attention</a></td><td><code>ring_degree > 1</code>, <code>ring_anything=True</code></td><td><code>ulysses_degree</code> harus 1, hanya untuk inferensi, <code>attn_mask</code> harus <code>None</code></td><td>panjang sekuens tidak terbagi rata</td></tr></tbody></table> <!--[1--><h2 class="relative group"><a id="ring-attention" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#ring-attention"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Ring Attention<!----></span></h2><!--]--><!----> <p><a href="https://huggingface.co/papers/2310.01889" rel="nofollow">Ring Attention</a> meneruskan blok key (K) dan value (V) antar-GPU yang disusun dalam bentuk cincin. Setiap GPU menghitung perhatian terhadap blok K/V yang dipegangnya, lalu meneruskan blok tersebut ke GPU berikutnya. Setelah satu putaran penuh, setiap GPU telah melihat K/V dari setiap token, dan tidak ada GPU yang pernah memegang seluruh rangkaian.</p> <p>Set <code>ring_degree</code> ke jumlah GPU, lalu lewati config ke metode <code>enable_parallelism()</code> transformer.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Disalin</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">import</span> os
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> torch <span class="hljs-keyword">import</span> distributed <span class="hljs-keyword">as</span> dist
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> DiffusionPipeline, ContextParallelConfig
<span class="hljs-keyword">def</span> <span class="hljs-title function_">setup_distributed</span>():
<span class="hljs-keyword">if</span> <span class="hljs-keyword">not</span> dist.is_initialized():
dist.init_process_group(backend=<span class="hljs-string">&quot;nccl&quot;</span>)
<span class="hljs-comment"># torchrun sets LOCAL_RANK to this process&#x27;s GPU index on its node</span>
local_rank = <span class="hljs-built_in">int</span>(os.environ[<span class="hljs-string">&quot;LOCAL_RANK&quot;</span>])
device = torch.device(<span class="hljs-string">f&quot;cuda:<span class="hljs-subst">{local_rank}</span>&quot;</span>)
torch.cuda.set_device(device)
<span class="hljs-keyword">return</span> device
<span class="hljs-keyword">def</span> <span class="hljs-title function_">main</span>():
device = setup_distributed()
world_size = dist.get_world_size()
pipeline = DiffusionPipeline.from_pretrained(
<span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>, dtype=torch.bfloat16
).to(device)
pipeline.transformer.set_attention_backend(<span class="hljs-string">&quot;_native_cudnn&quot;</span>)
cp_config = ContextParallelConfig(ring_degree=world_size)
pipeline.transformer.enable_parallelism(config=cp_config)
prompt = <span class="hljs-string">&quot;&quot;&quot;
cinematic film still of a cat sipping a margarita in a pool in Palm Springs, California
highly detailed, high budget hollywood movie, cinemascope, moody, epic, gorgeous, film grain
&quot;&quot;&quot;</span>
<span class="hljs-comment"># Must specify generator so all ranks start with same latents (or pass your own)</span>
generator = torch.Generator().manual_seed(<span class="hljs-number">42</span>)
image = pipeline(
prompt,
guidance_scale=<span class="hljs-number">3.5</span>,
num_inference_steps=<span class="hljs-number">50</span>,
generator=generator,
).images[<span class="hljs-number">0</span>]
<span class="hljs-keyword">if</span> dist.get_rank() == <span class="hljs-number">0</span>:
image.save(<span class="hljs-string">&quot;output.png&quot;</span>)
<span class="hljs-keyword">if</span> dist.is_initialized():
dist.destroy_process_group()
<span class="hljs-keyword">if</span> __name__ == <span class="hljs-string">&quot;__main__&quot;</span>:
main()<!----></pre></div><!----> <p>Simpan skrip sebagai <code>ring_attention.py</code> dan jalankan dengan <a href="https://docs.pytorch.org/docs/stable/elastic/run.html" rel="nofollow">torchrun</a> atau launcher lain yang kompatibel dengan PyTorch. Atur <code>--nproc-per-node</code> ke jumlah GPU.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Disalin</div><!----></button><!----></div> <pre class="language-bash "><!---->torchrun --nproc-per-node 2 ring_attention.py<!----></pre></div><!----> <p>Untuk mencoba varian lainnya, ganti panggilan <code>enable_parallelism</code> di <code>ring_attention.py</code> dengan yang ada di bagian mereka.</p> <!--[1--><h2 class="relative group"><a id="ulysses-attention" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#ulysses-attention"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Perhatian Ulysses<!----></span></h2><!--]--><!----> <p><a href="https://huggingface.co/papers/2309.14509" rel="nofollow">Ulysses Attention</a> menggunakan komunikasi <em>all-to-all</em>, di mana setiap GPU bertukar data dengan setiap GPU lainnya. all-to-all pertama mengelompokkan ulang data sehingga setiap GPU memegang setiap token untuk subset dari attention heads. Setiap GPU menghitung attention untuk heads-nya, dan all-to-all kedua mengelompokkan ulang hasil berdasarkan token untuk layer berikutnya.</p> <p>Set <code>ulysses_degree</code> ke jumlah GPU, dan lewati config ke <code>enable_parallelism()</code>.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Disalin</div><!----></button><!----></div> <pre class="language-py "><!---->pipeline.transformer.enable_parallelism(config=ContextParallelConfig(ulysses_degree=<span class="hljs-number">2</span>))<!----></pre></div><!----> <!--[1--><h2 class="relative group"><a id="unified-attention" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#unified-attention"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Perhatian Terpadu<!----></span></h2><!--]--><!----> <p><a href="https://huggingface.co/papers/2405.07719" rel="nofollow">Unified Sequence Parallelism</a> menggabungkan Ring Attention dan Ulysses Attention. Proses ini menjalankan all-to-all untuk mendistribusikan ulang heads dan tokens, kemudian menjalankan Ring Attention pada hasilnya, lalu membalikkan all-to-all untuk memulihkan tata letak asli. Ulysses melakukan paralelisasi di antara attention heads, sedangkan Ring melakukan paralelisasi di sepanjang sequence, sehingga secara bersama-sama keduanya dapat menggunakan lebih banyak GPU dibandingkan Ulysses saja.</p> <p>Gunakan Unified Attention ketika Anda memiliki setidaknya 4 GPU. Atur <code>ulysses_degree</code> dan <code>ring_degree</code> keduanya ke nilai lebih dari 1. GPU membentuk grid 2D dari <code>ulysses_degree * ring_degree</code> perangkat, dan setiap GPU termasuk dalam satu grup Ulysses dan satu grup Ring.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Disalin</div><!----></button><!----></div> <pre class="language-py "><!---->pipeline.transformer.enable_parallelism(config=ContextParallelConfig(ulysses_degree=<span class="hljs-number">2</span>, ring_degree=<span class="hljs-number">2</span>))<!----></pre></div><!----> <p>Benchmark di bawah ini membandingkan Ulysses, Ring, dan Unified Attention pada 4 GPU H100 dengan <a href="https://github.com/huggingface/diffusers/pull/12693#issuecomment-3694727532" rel="nofollow">skrip ini</a>. Setiap gambar menggunakan 50 langkah inferensi.</p> <table><thead><tr><th>Backend CP</th><th>Waktu / Gambar (ms)</th><th>Langkah / Detik</th><th>Memori Puncak (GB)</th></tr></thead><tbody><tr><td>ulysses</td><td>6670.789</td><td>7.50</td><td>33.85</td></tr><tr><td>ring</td><td>13076.492</td><td>3.82</td><td>56.02</td></tr><tr><td>unified_balanced</td><td>11068.705</td><td>4.52</td><td>33.85</td></tr></tbody></table> <p>Ulysses memiliki throughput tertinggi, tetapi tidak dapat menggunakan lebih banyak GPU daripada jumlah kepala perhatian (attention heads) pada model. Unified Attention menghilangkan batasan tersebut.</p> <!--[1--><h2 class="relative group"><a id="ulysses-anything-attention" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#ulysses-anything-attention"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Ulysses Anything Attention<!----></span></h2><!--]--><!----> <p>Ulysses Attention membutuhkan panjang sekuens dan jumlah kepala perhatian yang dapat dibagi rata oleh <code>ulysses_degree</code>. <a href="https://github.com/huggingface/diffusers/pull/12996" rel="nofollow">Ulysses Anything Attention</a> menghilangkan kedua persyaratan tersebut. Anda tidak dapat menggabungkannya dengan Unified Attention, sehingga <code>ring_degree</code> harus bernilai 1.</p> <p>Set <code>ulysses_degree</code> ke lebih dari 1 dan <code>ulysses_anything=True</code>.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Disalin</div><!----></button><!----></div> <pre class="language-py "><!---->pipeline.transformer.enable_parallelism(config=ContextParallelConfig(ulysses_degree=<span class="hljs-number">2</span>, ulysses_anything=<span class="hljs-literal">True</span>))<!----></pre></div><!----> <blockquote class="tip"><p>Tambahkan backend <code>gloo</code> untuk tensor CPU ke <a href="https://docs.pytorch.org/docs/stable/distributed.html#torch.distributed.init_process_group" rel="nofollow">init_process_group</a>. Ini menghindari sinkronisasi CUDA yang dipaksakan akibat transfer dari host ke perangkat dan dari perangkat ke host.</p></blockquote> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Disalin</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">import</span> torch.distributed <span class="hljs-keyword">as</span> dist
dist.init_process_group(backend=<span class="hljs-string">&quot;cpu:gloo,cuda:nccl&quot;</span>)<!----></pre></div><!----> <p>Benchmark di bawah ini menjalankan FLUX.1-dev pada 4 GPU L20 dengan <a href="https://github.com/huggingface/diffusers/pull/12996#issuecomment-3797695999" rel="nofollow">skrip ini</a>.</p> <table><thead><tr><th>Backend CP</th><th>Waktu / Iterasi (ms)</th><th>Langkah / Detik</th><th>Memori Puncak (GB)</th><th>Bentuk (HxW)</th></tr></thead><tbody><tr><td>ulysses</td><td>281.07</td><td>3.56</td><td>37.11</td><td>1024x1024</td></tr><tr><td>ring</td><td>351.34</td><td>2.85</td><td>37.01</td><td>1024x1024</td></tr><tr><td>unified_balanced</td><td>324.37</td><td>3.08</td><td>37.16</td><td>1024x1024</td></tr><tr><td>ulysses_anything</td><td>280.94</td><td>3.56</td><td>37.11</td><td>1024x1024</td></tr><tr><td>ulysses</td><td>gagal</td><td>gagal</td><td>gagal</td><td>1008x1008</td></tr><tr><td>ring</td><td>gagal</td><td>gagal</td><td>gagal</td><td>1008x1008</td></tr><tr><td>unified_balanced</td><td>gagal</td><td>gagal</td><td>gagal</td><td>1008x1008</td></tr><tr><td>ulysses_anything</td><td>278.40</td><td>3.59</td><td>36.99</td><td>1008x1008</td></tr></tbody></table> <p>Pada 1008x1008, di mana panjang sekuens tidak terbagi rata, hanya Ulysses Anything Attention yang berjalan. Pada 1024x1024, kinerjanya sama dengan Ulysses Attention standar.</p> <!--[1--><h2 class="relative group"><a id="ring-anything-attention" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#ring-anything-attention"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Ring Anything Attention<!----></span></h2><!--]--><!----> <p>Ring Attention membutuhkan panjang sekuens yang dapat dibagi rata oleh <code>ring_degree</code>. <a href="https://github.com/huggingface/diffusers/pull/13545#issuecomment-4302195582" rel="nofollow">Ring Anything Attention</a> mendukung panjang sekuens apa pun. Ia melakukan padding pada K/V lokal di setiap GPU hingga panjang lokal terpanjang, melakukan all-gather pada K/V yang sudah di-padding, dan memotong kembali setiap blok ke panjang aslinya sebelum menghitung attention.</p> <p>Anda tidak dapat menggabungkannya dengan Unified Attention, sehingga <code>ulysses_degree</code> harus 1. Atur <code>ring_degree</code> ke lebih dari 1 dan <code>ring_anything=True</code>.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Disalin</div><!----></button><!----></div> <pre class="language-py "><!---->pipeline.transformer.enable_parallelism(config=ContextParallelConfig(ring_degree=<span class="hljs-number">2</span>, ring_anything=<span class="hljs-literal">True</span>))<!----></pre></div><!----> <p>Ring Anything Attention juga mendapat manfaat dari backend <code>gloo</code> yang ditampilkan di <a href="#ulysses-anything-attention">Ulysses Anything Attention</a>.</p> <blockquote class="note"><p>Ring Anything Attention hanya mendukung inferensi, dan <code>attn_mask</code> harus <code>None</code>.</p></blockquote> <p>Benchmarks di bawah ini menjalankan FLUX.1-dev pada 4 GPU RTX 4090 (48GB).</p> <table><thead><tr><th>Backend CP</th><th>Waktu / Iterasi (ms)</th><th>Langkah / Detik</th><th>Memori Puncak (GB)</th><th>Bentuk (HxW)</th></tr></thead><tbody><tr><td>ulysses</td><td>259.07</td><td>3.86</td><td>33.83</td><td>1024x1024</td></tr><tr><td>ring</td><td>338.98</td><td>2.95</td><td>33.83</td><td>1024x1024</td></tr><tr><td>unified_balanced</td><td>321.54</td><td>3.11</td><td>33.83</td><td>1024x1024</td></tr><tr><td>ulysses_anything</td><td>259.07</td><td>3.86</td><td>33.83</td><td>1024x1024</td></tr><tr><td>ring_anything</td><td>340.14</td><td>2.94</td><td>33.83</td><td>1024x1024</td></tr><tr><td>ulysses</td><td>gagal</td><td>gagal</td><td>gagal</td><td>1008x1008</td></tr><tr><td>ring</td><td>gagal</td><td>gagal</td><td>gagal</td><td>1008x1008</td></tr><tr><td>unified_balanced</td><td>gagal</td><td>gagal</td><td>gagal</td><td>1008x1008</td></tr><tr><td>ulysses_anything</td><td>253.16</td><td>3.95</td><td>33.75</td><td>1008x1008</td></tr><tr><td>ring_anything</td><td>335.57</td><td>2.98</td><td>33.75</td><td>1008x1008</td></tr></tbody></table> <p>Ring Anything Attention berjalan pada 1008x1008 dan kinerjanya kurang lebih sama dengan Ring Attention standar. Untuk motivasi dan trade-off-nya, lihat <a href="https://github.com/huggingface/diffusers/pull/13545#issuecomment-4304104462" rel="nofollow">komentar ini</a>.</p> <!--[1--><h2 class="relative group"><a id="enable-context-parallelism-while-loading" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#enable-context-parallelism-while-loading"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Aktifkan paralelisme konteks saat memuat<!----></span></h2><!--]--><!----> <p>Alih-alih memanggil <code>enable_parallelism</code> setelah memuat, teruskan <a href="/docs/diffusers/v0.43.0/id/api/parallel#diffusers.ContextParallelConfig">ContextParallelConfig</a> ke argumen <code>parallel_config</code> dari <a href="/docs/diffusers/v0.43.0/id/api/models/overview#diffusers.ModelMixin.from_pretrained">from_pretrained()</a>. Inisialisasi grup proses terlebih dahulu, karena <code>from_pretrained</code> akan melempar error jika <code>torch.distributed</code> belum diinisialisasi. Jalankan skrip dengan <code>torchrun</code>, seperti pada contoh <a href="#ring-attention">Ring Attention</a>.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Disalin</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">import</span> os
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> torch <span class="hljs-keyword">import</span> distributed <span class="hljs-keyword">as</span> dist
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> AutoModel, ContextParallelConfig, DiffusionPipeline
CKPT_ID = <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>
dist.init_process_group(backend=<span class="hljs-string">&quot;nccl&quot;</span>)
local_rank = <span class="hljs-built_in">int</span>(os.environ[<span class="hljs-string">&quot;LOCAL_RANK&quot;</span>])
device = torch.device(<span class="hljs-string">f&quot;cuda:<span class="hljs-subst">{local_rank}</span>&quot;</span>)
torch.cuda.set_device(device)
cp_config = ContextParallelConfig(ring_degree=dist.get_world_size())
transformer = AutoModel.from_pretrained(
CKPT_ID,
subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
dtype=torch.bfloat16,
parallel_config=cp_config
)
pipeline = DiffusionPipeline.from_pretrained(
CKPT_ID, transformer=transformer, dtype=torch.bfloat16,
).to(device)<!----></pre></div><!----> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/diffusers/blob/main/docs/source/en/optimization/context_parallelism.md" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg><!----> <span><span class="underline">Update</span> on GitHub</span></a><!----> <p></p><!--]--><!----><!--]--><!--]--><!--]--> <!--[-1--><!--]--><!--]-->
<script>
{
__sveltekit_1qsdbwf = {
base: "/docs/diffusers/main/en",
assets: "/docs/diffusers/main/en"
};
const element = document.currentScript.parentElement;
Promise.all([
import("/docs/diffusers/main/en/_app/immutable/entry/start.0S0_iEoG.js"),
import("/docs/diffusers/main/en/_app/immutable/entry/app.CPdOxCbw.js")
]).then(([kit, app]) => {
kit.start(app, element, {
node_ids: [0, 333],
data: [null,null],
form: null,
error: null
});
});
}
</script>

Xet Storage Details

Size:
40.5 kB
·
Xet hash:
2b2651abc049c989226172b20414374bf6cf8c361fbca501be5cb8fd42482107

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.