Buckets:

hf-doc-build
/

doc

Files
xet
hf-doc-build/doc / diffusers /main /fr /optimization /context_parallelism.html
mishig
3 days ago
download
raw
40.8 kB
<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;Parallélisme de contexte&quot;,&quot;local&quot;:&quot;context-parallelism&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Ring Attention&quot;,&quot;local&quot;:&quot;ring-attention&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Ulysses Attention&quot;,&quot;local&quot;:&quot;ulysses-attention&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Attention unifiée&quot;,&quot;local&quot;:&quot;unified-attention&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Ulysses Anything Attention&quot;,&quot;local&quot;:&quot;ulysses-anything-attention&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Ring Anything Attention&quot;,&quot;local&quot;:&quot;ring-anything-attention&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Activer le parallélisme de contexte lors du chargement&quot;,&quot;local&quot;:&quot;enable-context-parallelism-while-loading&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}"/>
		<link href="/docs/diffusers/main/en/_app/immutable/entry/start.0S0_iEoG.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/I9eooN77.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/Bupkjr9f.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/entry/app.CPdOxCbw.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/CyvZ3FS5.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/l0QnnOED.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/DnVIJfC5.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/DsnmJJEf.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/nodes/0.DteTvmQZ.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/CoSrdQax.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/nodes/333.OuTuM9lH.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/DSPDl1gz.js" rel="modulepreload">
		<!--ty2c99--><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;Context parallelism&quot;,&quot;local&quot;:&quot;context-parallelism&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Ring Attention&quot;,&quot;local&quot;:&quot;ring-attention&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Ulysses Attention&quot;,&quot;local&quot;:&quot;ulysses-attention&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Unified Attention&quot;,&quot;local&quot;:&quot;unified-attention&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Ulysses Anything Attention&quot;,&quot;local&quot;:&quot;ulysses-anything-attention&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Ring Anything Attention&quot;,&quot;local&quot;:&quot;ring-anything-attention&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Enable context parallelism while loading&quot;,&quot;local&quot;:&quot;enable-context-parallelism-while-loading&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}"/><!---->
		<link href="/docs/diffusers/main/en/_app/immutable/assets/0.tn0RQdqM.css" rel="modulepreload"> <!--[--><!--[0--><!--[--><!--[0--><!--[--><p></p> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: inline-end; margin-inline-start: 10px; display: inline-flex; position: relative; z-index: 10;"><div dir="ltr" class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 h-7 max-sm:h-7 px-2 max-sm:px-1.5 text-sm font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0 hover:text-gray-800 dark:hover:text-gray-200"><svg class="sm:size-3.5 size-3" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----></span> <span dir="auto">Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-7 max-sm:h-7 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible sm:size-3.5 size-3 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg><!----></button></div> <!--[-1--><!--]--></div><!----> <!--[0--><h1 class="relative group"><a id="context-parallelism" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#context-parallelism"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Parallélisme de contexte<!----></span></h1><!--]--><!----> <p><a href="https://huggingface.co/spaces/nanotron/ultrascale-playbook?section=context_parallelism" rel="nofollow">Le parallélisme de contexte</a> répartit la séquence d’entrée sur plusieurs GPU, et chaque GPU traite sa propre portion. Cela réduit la mémoire par GPU, ce qui est particulièrement utile pour les séquences longues, comme les images haute résolution et les vidéos. Pour le comparer aux autres stratégies, consultez <a href="../training/distributed_inference">Inférence distribuée</a>.</p> <p>Le parallélisme de contexte s’exécute à l’intérieur du backend d’attention, et seuls certains backends le prennent en charge. Si le vôtre ne le fait pas, <code>enable_parallelism</code> lève une erreur qui énumère les backends compatibles. Changez de backend avec <a href="/docs/diffusers/main/fr/api/models/overview#diffusers.ModelMixin.set_attention_backend">set_attention_backend()</a> et consultez les <a href="./attention_backends#available-backends">backends disponibles</a> pour la liste complète.</p> <p>Configurez chaque variante ci-dessous avec <a href="/docs/diffusers/main/fr/api/parallel#diffusers.ContextParallelConfig">ContextParallelConfig</a>.</p> <table><thead><tr><th>Variante</th><th>Configuration</th><th>Prérequis</th><th>À utiliser lorsque</th></tr></thead><tbody><tr><td><a href="#ring-attention">Ring Attention</a></td><td><code>ring_degree > 1</code></td><td>la longueur de la séquence doit être divisible par <code>ring_degree</code></td><td>les séquences sont longues et la bande passante d'interconnexion est limitée</td></tr><tr><td><a href="#ulysses-attention">Attention Ulysses</a></td><td><code>ulysses_degree > 1</code></td><td>la longueur de la séquence et le nombre de têtes d'attention doivent être divisibles par <code>ulysses_degree</code></td><td>vos GPU disposent d'un interconnect rapide</td></tr><tr><td><a href="#unified-attention">Attention unifiée</a></td><td><code>ring_degree > 1</code> et <code>ulysses_degree > 1</code></td><td>au moins 4 GPU</td><td>vous disposez de plus d'appareils que l'attention Ulysses peut utiliser seul</td></tr><tr><td><a href="#ulysses-anything-attention">Ulysses Anything Attention</a></td><td><code>ulysses_degree > 1</code>, <code>ulysses_anything=True</code></td><td><code>ring_degree</code> doit être 1</td><td>la longueur de la séquence ou le nombre de têtes ne se divise pas exactement</td></tr><tr><td><a href="#ring-anything-attention">Ring Anything Attention</a></td><td><code>ring_degree > 1</code>, <code>ring_anything=True</code></td><td><code>ulysses_degree</code> doit être 1, inférence uniquement, <code>attn_mask</code> doit être <code>None</code></td><td>la longueur de la séquence n’est pas divisible</td></tr></tbody></table> <!--[1--><h2 class="relative group"><a id="ring-attention" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#ring-attention"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Ring Attention<!----></span></h2><!--]--><!----> <p><a href="https://huggingface.co/papers/2310.01889" rel="nofollow">Ring Attention</a> transmet les blocs de clés (K) et de valeurs (V) entre des GPU disposés en anneau. Chaque GPU calcule l'attention par rapport au bloc K/V qu'il détient, puis transmet ce bloc au GPU suivant. Après un tour complet, chaque GPU a vu les K/V de tous les jetons, et aucun GPU ne détient jamais la séquence complète.</p> <p>Définissez <code>ring_degree</code> sur le nombre de GPU et passez la configuration à la méthode <code>enable_parallelism()</code> du transformeur.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copié</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">import</span> os
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> torch <span class="hljs-keyword">import</span> distributed <span class="hljs-keyword">as</span> dist
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> DiffusionPipeline, ContextParallelConfig

<span class="hljs-keyword">def</span> <span class="hljs-title function_">setup_distributed</span>():
    <span class="hljs-keyword">if</span> <span class="hljs-keyword">not</span> dist.is_initialized():
        dist.init_process_group(backend=<span class="hljs-string">&quot;nccl&quot;</span>)
    <span class="hljs-comment"># torchrun sets LOCAL_RANK to this process&#x27;s GPU index on its node</span>
    local_rank = <span class="hljs-built_in">int</span>(os.environ[<span class="hljs-string">&quot;LOCAL_RANK&quot;</span>])
    device = torch.device(<span class="hljs-string">f&quot;cuda:<span class="hljs-subst">{local_rank}</span>&quot;</span>)
    torch.cuda.set_device(device)
    <span class="hljs-keyword">return</span> device

<span class="hljs-keyword">def</span> <span class="hljs-title function_">main</span>():
    device = setup_distributed()
    world_size = dist.get_world_size()

    pipeline = DiffusionPipeline.from_pretrained(
        <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>, dtype=torch.bfloat16
    ).to(device)
    pipeline.transformer.set_attention_backend(<span class="hljs-string">&quot;_native_cudnn&quot;</span>)

    cp_config = ContextParallelConfig(ring_degree=world_size)
    pipeline.transformer.enable_parallelism(config=cp_config)

    prompt = <span class="hljs-string">&quot;&quot;&quot;
    cinematic film still of a cat sipping a margarita in a pool in Palm Springs, California
    highly detailed, high budget hollywood movie, cinemascope, moody, epic, gorgeous, film grain
    &quot;&quot;&quot;</span>

    <span class="hljs-comment"># Must specify generator so all ranks start with same latents (or pass your own)</span>
    generator = torch.Generator().manual_seed(<span class="hljs-number">42</span>)
    image = pipeline(
        prompt,
        guidance_scale=<span class="hljs-number">3.5</span>,
        num_inference_steps=<span class="hljs-number">50</span>,
        generator=generator,
    ).images[<span class="hljs-number">0</span>]

    <span class="hljs-keyword">if</span> dist.get_rank() == <span class="hljs-number">0</span>:
        image.save(<span class="hljs-string">&quot;output.png&quot;</span>)

    <span class="hljs-keyword">if</span> dist.is_initialized():
        dist.destroy_process_group()


<span class="hljs-keyword">if</span> __name__ == <span class="hljs-string">&quot;__main__&quot;</span>:
    main()<!----></pre></div><!----> <p>Enregistrez le script sous le nom <code>ring_attention.py</code> et lancez-le avec <a href="https://docs.pytorch.org/docs/stable/elastic/run.html" rel="nofollow">torchrun</a> ou un autre lanceur compatible PyTorch. Définissez <code>--nproc-per-node</code> sur le nombre de GPU.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copié</div><!----></button><!----></div> <pre class="language-bash "><!---->torchrun --nproc-per-node 2 ring_attention.py<!----></pre></div><!----> <p>Pour essayer les autres variantes, remplacez l'appel <code>enable_parallelism</code> dans <code>ring_attention.py</code> par celui de leur section.</p> <!--[1--><h2 class="relative group"><a id="ulysses-attention" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#ulysses-attention"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Ulysses Attention<!----></span></h2><!--]--><!----> <p><a href="https://huggingface.co/papers/2309.14509" rel="nofollow">Ulysses Attention</a> utilise la communication <em>all-to-all</em>, où chaque GPU échange des données avec tous les autres GPU. Le premier all-to-all regroupe les données de sorte que chaque GPU détient tous les jetons pour un sous-ensemble de têtes d'attention. Chaque GPU calcule l'attention pour ses têtes, et un second all-to-all regroupe les résultats par jeton pour la couche suivante.</p> <p>Définissez <code>ulysses_degree</code> sur le nombre de GPU et passez la configuration à <code>enable_parallelism()</code>.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copié</div><!----></button><!----></div> <pre class="language-py "><!---->pipeline.transformer.enable_parallelism(config=ContextParallelConfig(ulysses_degree=<span class="hljs-number">2</span>))<!----></pre></div><!----> <!--[1--><h2 class="relative group"><a id="unified-attention" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#unified-attention"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Attention unifiée<!----></span></h2><!--]--><!----> <p><a href="https://huggingface.co/papers/2405.07719" rel="nofollow">Unified Sequence Parallelism</a> combine Ring Attention et Ulysses Attention. Il exécute un all-to-all pour redistribuer les têtes et les jetons, applique Ring Attention au résultat, puis inverse l'all-to-all pour restaurer la disposition d'origine. Ulysses parallélise sur les têtes d'attention et Ring sur la séquence, ce qui leur permet, ensemble, d'utiliser plus de GPU qu'Ulysses seul.</p> <p>Utilisez Unified Attention lorsque vous disposez d’au moins 4 GPU. Définissez à la fois <code>ulysses_degree</code> et <code>ring_degree</code> sur une valeur supérieure à 1. Les GPU forment une grille 2D de <code>ulysses_degree * ring_degree</code> dispositifs, et chaque GPU appartient à un groupe Ulysses et à un groupe Ring.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copié</div><!----></button><!----></div> <pre class="language-py "><!---->pipeline.transformer.enable_parallelism(config=ContextParallelConfig(ulysses_degree=<span class="hljs-number">2</span>, ring_degree=<span class="hljs-number">2</span>))<!----></pre></div><!----> <p>Le benchmark ci-dessous compare Ulysses, Ring et Unified Attention sur 4 GPU H100 à l'aide de <a href="https://github.com/huggingface/diffusers/pull/12693#issuecomment-3694727532" rel="nofollow">ce script</a>. Chaque image utilise 50 étapes d'inférence.</p> <table><thead><tr><th>Backend CP</th><th>Temps / Image (ms)</th><th>Étapes / Sec</th><th>Mémoire maximale (Go)</th></tr></thead><tbody><tr><td>ulysses</td><td>6670.789</td><td>7.50</td><td>33.85</td></tr><tr><td>ring</td><td>13076.492</td><td>3.82</td><td>56.02</td></tr><tr><td>unified_balanced</td><td>11068.705</td><td>4.52</td><td>33.85</td></tr></tbody></table> <p>Ulysses offre le débit le plus élevé, mais il ne peut pas utiliser plus de GPU que le nombre de têtes d'attention du modèle. Unified Attention supprime cette limite.</p> <!--[1--><h2 class="relative group"><a id="ulysses-anything-attention" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#ulysses-anything-attention"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Ulysses Anything Attention<!----></span></h2><!--]--><!----> <p>L’attention Ulysses exige que la longueur de la séquence et le nombre de têtes d’attention soient divisibles par <code>ulysses_degree</code>. <a href="https://github.com/huggingface/diffusers/pull/12996" rel="nofollow">Ulysses Anything Attention</a> supprime ces deux contraintes. Il ne peut pas être combiné avec Unified Attention, donc <code>ring_degree</code> doit être égal à 1.</p> <p>Définissez <code>ulysses_degree</code> sur une valeur supérieure à 1 et <code>ulysses_anything=True</code>.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copié</div><!----></button><!----></div> <pre class="language-py "><!---->pipeline.transformer.enable_parallelism(config=ContextParallelConfig(ulysses_degree=<span class="hljs-number">2</span>, ulysses_anything=<span class="hljs-literal">True</span>))<!----></pre></div><!----> <blockquote class="tip"><p>Ajoutez un backend <code>gloo</code> pour les tenseurs CPU à <a href="https://docs.pytorch.org/docs/stable/distributed.html#torch.distributed.init_process_group" rel="nofollow">init_process_group</a>. Cela évite les synchronisations CUDA forcées causées par les transferts hôte-vers-appareil et appareil-vers-hôte.</p></blockquote> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copié</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">import</span> torch.distributed <span class="hljs-keyword">as</span> dist

dist.init_process_group(backend=<span class="hljs-string">&quot;cpu:gloo,cuda:nccl&quot;</span>)<!----></pre></div><!----> <p>Le benchmark ci-dessous exécute FLUX.1-dev sur 4 GPU L20 avec <a href="https://github.com/huggingface/diffusers/pull/12996#issuecomment-3797695999" rel="nofollow">ce script</a>.</p> <table><thead><tr><th>Backend CP</th><th>Temps / Itération (ms)</th><th>Étapes / Sec</th><th>Mémoire maximale (Go)</th><th>Forme (HxW)</th></tr></thead><tbody><tr><td>ulysses</td><td>281.07</td><td>3.56</td><td>37.11</td><td>1024x1024</td></tr><tr><td>ring</td><td>351.34</td><td>2.85</td><td>37.01</td><td>1024x1024</td></tr><tr><td>unified_balanced</td><td>324.37</td><td>3.08</td><td>37.16</td><td>1024x1024</td></tr><tr><td>ulysses_anything</td><td>280.94</td><td>3.56</td><td>37.11</td><td>1024x1024</td></tr><tr><td>ulysses</td><td>échoué</td><td>échoué</td><td>échoué</td><td>1008x1008</td></tr><tr><td>ring</td><td>échoué</td><td>échoué</td><td>échoué</td><td>1008x1008</td></tr><tr><td>unified_balanced</td><td>échoué</td><td>échoué</td><td>échoué</td><td>1008x1008</td></tr><tr><td>ulysses_anything</td><td>278.40</td><td>3.59</td><td>36.99</td><td>1008x1008</td></tr></tbody></table> <p>À 1008x1008, où la longueur de la séquence ne se divise pas exactement, seul Ulysses Anything Attention s’exécute. À 1024x1024, il offre des performances identiques à celles de l’Ulysses Attention standard.</p> <!--[1--><h2 class="relative group"><a id="ring-anything-attention" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#ring-anything-attention"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Ring Anything Attention<!----></span></h2><!--]--><!----> <p>Ring Attention exige que la longueur de la séquence soit divisible par <code>ring_degree</code>. <a href="https://github.com/huggingface/diffusers/pull/13545#issuecomment-4302195582" rel="nofollow">Ring Anything Attention</a> prend en charge toute longueur de séquence. Il remplit les K/V locaux de chaque GPU jusqu'à la longueur locale maximale, effectue un all-gather sur les K/V remplis, puis recoupe chaque bloc à sa longueur réelle avant de calculer l'attention.</p> <p>Vous ne pouvez pas le combiner avec Unified Attention, donc <code>ulysses_degree</code> doit être 1. Définissez <code>ring_degree</code> sur une valeur supérieure à 1 et <code>ring_anything=True</code>.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copié</div><!----></button><!----></div> <pre class="language-py "><!---->pipeline.transformer.enable_parallelism(config=ContextParallelConfig(ring_degree=<span class="hljs-number">2</span>, ring_anything=<span class="hljs-literal">True</span>))<!----></pre></div><!----> <p>Ring Anything Attention bénéficie également de l’arrière-plan <code>gloo</code> présenté dans <a href="#ulysses-anything-attention">Ulysses Anything Attention</a>.</p> <blockquote class="note"><p>Ring Anything Attention prend en charge uniquement l'inférence, et <code>attn_mask</code> doit être <code>None</code>.</p></blockquote> <p>Le benchmark ci-dessous exécute FLUX.1-dev sur 4 GPU RTX 4090 (48 Go).</p> <table><thead><tr><th>Backend CP</th><th>Temps / Itération (ms)</th><th>Étapes / Sec</th><th>Mémoire maximale (Go)</th><th>Forme (HxW)</th></tr></thead><tbody><tr><td>ulysses</td><td>259.07</td><td>3.86</td><td>33.83</td><td>1024x1024</td></tr><tr><td>ring</td><td>338.98</td><td>2.95</td><td>33.83</td><td>1024x1024</td></tr><tr><td>unified_balanced</td><td>321.54</td><td>3.11</td><td>33.83</td><td>1024x1024</td></tr><tr><td>ulysses_anything</td><td>259.07</td><td>3.86</td><td>33.83</td><td>1024x1024</td></tr><tr><td>ring_anything</td><td>340.14</td><td>2.94</td><td>33.83</td><td>1024x1024</td></tr><tr><td>ulysses</td><td>échoué</td><td>échoué</td><td>échoué</td><td>1008x1008</td></tr><tr><td>ring</td><td>échoué</td><td>échoué</td><td>échoué</td><td>1008x1008</td></tr><tr><td>unified_balanced</td><td>échoué</td><td>échoué</td><td>échoué</td><td>1008x1008</td></tr><tr><td>ulysses_anything</td><td>253.16</td><td>3.95</td><td>33.75</td><td>1008x1008</td></tr><tr><td>ring_anything</td><td>335.57</td><td>2.98</td><td>33.75</td><td>1008x1008</td></tr></tbody></table> <p>Ring Anything Attention s’exécute à 1008x1008 et offre des performances similaires à celles de Ring Attention standard. Pour la motivation et les compromis, consultez <a href="https://github.com/huggingface/diffusers/pull/13545#issuecomment-4304104462" rel="nofollow">ce commentaire</a>.</p> <!--[1--><h2 class="relative group"><a id="enable-context-parallelism-while-loading" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#enable-context-parallelism-while-loading"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Activer le parallélisme de contexte lors du chargement<!----></span></h2><!--]--><!----> <p>Au lieu d’appeler <code>enable_parallelism</code> après le chargement, passez le <a href="/docs/diffusers/main/fr/api/parallel#diffusers.ContextParallelConfig">ContextParallelConfig</a> à l’argument <code>parallel_config</code> de <a href="/docs/diffusers/main/fr/api/models/overview#diffusers.ModelMixin.from_pretrained">from_pretrained()</a>. Initialisez d’abord le groupe de processus, car <code>from_pretrained</code> lève une erreur si <code>torch.distributed</code> n’est pas initialisé. Lancez le script avec <code>torchrun</code>, comme dans l’exemple <a href="#ring-attention">Ring Attention</a>.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copié</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">import</span> os
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> torch <span class="hljs-keyword">import</span> distributed <span class="hljs-keyword">as</span> dist
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> AutoModel, ContextParallelConfig, DiffusionPipeline

CKPT_ID = <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>

dist.init_process_group(backend=<span class="hljs-string">&quot;nccl&quot;</span>)
local_rank = <span class="hljs-built_in">int</span>(os.environ[<span class="hljs-string">&quot;LOCAL_RANK&quot;</span>])
device = torch.device(<span class="hljs-string">f&quot;cuda:<span class="hljs-subst">{local_rank}</span>&quot;</span>)
torch.cuda.set_device(device)

cp_config = ContextParallelConfig(ring_degree=dist.get_world_size())
transformer = AutoModel.from_pretrained(
    CKPT_ID,
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    dtype=torch.bfloat16,
    parallel_config=cp_config
)

pipeline = DiffusionPipeline.from_pretrained(
    CKPT_ID, transformer=transformer, dtype=torch.bfloat16,
).to(device)<!----></pre></div><!----> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/diffusers/blob/main/docs/source/en/optimization/context_parallelism.md" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg><!----> <span><span class="underline">Update</span> on GitHub</span></a><!----> <p></p><!--]--><!----><!--]--><!--]--><!--]--> <!--[-1--><!--]--><!--]-->
			
			<script>
				{
					__sveltekit_1qsdbwf = {
						base: "/docs/diffusers/main/en",
						assets: "/docs/diffusers/main/en"
					};
					const element = document.currentScript.parentElement;
					Promise.all([
						import("/docs/diffusers/main/en/_app/immutable/entry/start.0S0_iEoG.js"),
						import("/docs/diffusers/main/en/_app/immutable/entry/app.CPdOxCbw.js")
					]).then(([kit, app]) => {
						kit.start(app, element, {
							node_ids: [0, 333],
							data: [null,null],
							form: null,
							error: null
						});
					});
				}
			</script>
		
Xet Storage Details

Size:: 40.8 kB
Xet hash:: e4ad9e229b01fd6537921c3cfd0092126d66d28d2ac31d672ea4d08730cb2fad
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.