Buckets:
| <meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"Expert parallelism","local":"expert-parallelism","sections":[{"title":"DistributedConfig","local":"distributedconfig","sections":[{"title":"Choosing an implementation","local":"choosing-an-implementation","sections":[],"depth":3},{"title":"Adding an implementation","local":"adding-an-implementation","sections":[],"depth":3},{"title":"Overriding the plans","local":"overriding-the-plans","sections":[],"depth":3}],"depth":2},{"title":"Token dispatch","local":"token-dispatch","sections":[{"title":"How the sizes combine","local":"how-the-sizes-combine","sections":[],"depth":3},{"title":"With tensor parallelism","local":"with-tensor-parallelism","sections":[],"depth":3}],"depth":2},{"title":"Combining with FSDP2","local":"combining-with-fsdp2","sections":[],"depth":2},{"title":"API reference","local":"transformers.DistributedConfig","sections":[],"depth":2}],"depth":1}"/> | |
| <link href="/docs/transformers/pr_48566/en/_app/immutable/entry/start.GdwabqXz.js" rel="modulepreload"> | |
| <link href="/docs/transformers/pr_48566/en/_app/immutable/chunks/CFgBcTKX.js" rel="modulepreload"> | |
| <link href="/docs/transformers/pr_48566/en/_app/immutable/chunks/DdiSzEty.js" rel="modulepreload"> | |
| <link href="/docs/transformers/pr_48566/en/_app/immutable/entry/app.CrR-wy65.js" rel="modulepreload"> | |
| <link href="/docs/transformers/pr_48566/en/_app/immutable/chunks/DFjZUG2K.js" rel="modulepreload"> | |
| <link href="/docs/transformers/pr_48566/en/_app/immutable/chunks/Ctd2VdJv.js" rel="modulepreload"> | |
| <link href="/docs/transformers/pr_48566/en/_app/immutable/chunks/DsnmJJEf.js" rel="modulepreload"> | |
| <link href="/docs/transformers/pr_48566/en/_app/immutable/chunks/OQvPnLFT.js" rel="modulepreload"> | |
| <link href="/docs/transformers/pr_48566/en/_app/immutable/chunks/Cy46Z9Mw.js" rel="modulepreload"> | |
| <link href="/docs/transformers/pr_48566/en/_app/immutable/nodes/0.ClGKvxul.js" rel="modulepreload"> | |
| <link href="/docs/transformers/pr_48566/en/_app/immutable/chunks/Bry53OZs.js" rel="modulepreload"> | |
| <link href="/docs/transformers/pr_48566/en/_app/immutable/nodes/2.B5bDlV49.js" rel="modulepreload"> | |
| <link href="/docs/transformers/pr_48566/en/_app/immutable/chunks/CN3cOuQA.js" rel="modulepreload"> | |
| <!--9m8k7m--><meta name="hf:doc:metadata" content="{"title":"Expert parallelism","local":"expert-parallelism","sections":[{"title":"DistributedConfig","local":"distributedconfig","sections":[{"title":"Choosing an implementation","local":"choosing-an-implementation","sections":[],"depth":3},{"title":"Adding an implementation","local":"adding-an-implementation","sections":[],"depth":3},{"title":"Overriding the plans","local":"overriding-the-plans","sections":[],"depth":3}],"depth":2},{"title":"Token dispatch","local":"token-dispatch","sections":[{"title":"How the sizes combine","local":"how-the-sizes-combine","sections":[],"depth":3},{"title":"With tensor parallelism","local":"with-tensor-parallelism","sections":[],"depth":3}],"depth":2},{"title":"Combining with FSDP2","local":"combining-with-fsdp2","sections":[],"depth":2},{"title":"API reference","local":"transformers.DistributedConfig","sections":[],"depth":2}],"depth":1}"/><!----> | |
| <link href="/docs/transformers/pr_48566/en/_app/immutable/assets/0.tn0RQdqM.css" rel="modulepreload"> <!--[--><!--[0--><!--[--><!--[0--><!--[--><p></p> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: inline-end; margin-inline-start: 10px; display: inline-flex; position: relative; z-index: 10;"><div dir="ltr" class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 h-7 max-sm:h-7 px-2 max-sm:px-1.5 text-sm font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0 hover:text-gray-800 dark:hover:text-gray-200"><svg class="sm:size-3.5 size-3" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----></span> <span dir="auto">Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-7 max-sm:h-7 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible sm:size-3.5 size-3 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg><!----></button></div> <!--[-1--><!--]--></div><!----> <!--[0--><h1 class="relative group"><a id="expert-parallelism" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#expert-parallelism"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Expert parallelism<!----></span></h1><!--]--><!----> <p><a href="https://huggingface.co/spaces/nanotron/ultrascale-playbook?section=expert_parallelism" rel="nofollow">Expert parallelism</a> is a parallelism strategy for <a href="https://huggingface.co/blog/moe" rel="nofollow">mixture-of-experts (MoE) models</a>. Each expert’s feedforward layer lives on a different hardware accelerator. A router dispatches tokens to the appropriate experts and gathers the results. This approach scales models to far larger parameter counts without increasing computation cost because each token activates only a few experts.</p> <!--[1--><h2 class="relative group"><a id="distributedconfig" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#distributedconfig"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->DistributedConfig<!----></span></h2><!--]--><!----> <p>Enable expert parallelism with the <a href="/docs/transformers/pr_48566/en/expert_parallelism#transformers.DistributedConfig">DistributedConfig</a> class and the <code>ep_size</code> argument. MoE models shard their experts with <code>"ep_dispatch_experts"</code>. Its implementations pick how tokens reach the experts. <a href="#token-dispatch">Token dispatch</a> lets <code>ep_size</code> differ from <code>tp_size</code>. A few EP implementations require <code>ep_size=tp_size</code>, e.g. <code>"all_reduce"</code> and <code>"deepgemm_megamoe"</code>, so every rank of an expert group holds the same tokens.</p> <p>Llama 4 has no EP plan: its experts take every token already scaled by its router score, so there is nothing to dispatch. Shard them with tensor parallelism (<code>tp_size</code>) instead.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">import</span> os | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer | |
| <span class="hljs-keyword">from</span> transformers.distributed.configuration_utils <span class="hljs-keyword">import</span> DistributedConfig | |
| distributed_config = DistributedConfig( | |
| tp_size=<span class="hljs-built_in">int</span>(os.environ[<span class="hljs-string">"WORLD_SIZE"</span>]), | |
| ep_size=<span class="hljs-built_in">int</span>(os.environ[<span class="hljs-string">"WORLD_SIZE"</span>]), | |
| ) | |
| model = AutoModelForCausalLM.from_pretrained( | |
| <span class="hljs-string">"openai/gpt-oss-120b"</span>, | |
| distributed_config=distributed_config, | |
| )<!----></pre></div><!----> <p>Each MoE model defines two plans in its config: <code>base_model_tp_plan</code> for the dense modules and <code>base_model_ep_plan</code> for the experts. They are exposed on the loaded model as <code>model.tp_plan</code> and <code>model.ep_plan</code>. With <code>tp_size > 1</code> and <code>ep_size > 1</code>, both apply: the <a href="./perf_infer_gpu_multi">tensor parallel</a> plan shards attention and the dense MLPs, and the expert parallel plan shards the experts. EP rules take precedence over TP rules for the same modules, so expert weights are sharded once, by the EP plan. In the EP plan, the <code>GroupedGemmParallel</code> style splits the expert weights along the expert dimension so each rank loads only its local experts, and the <code>"ep_dispatch_experts"</code> rule on the experts module selects how tokens reach them.</p> <p><code>tp_plan</code> is applied only when <code>tp_size > 1</code>, and <code>ep_plan</code> only when <code>ep_size > 1</code>. With TP enabled and EP disabled, the full TP plan applies, expert rules included.</p> <p><code>"ep_dispatch_experts"</code> shards whole experts per rank (<code>ep_size</code> a multiple of <code>tp_size</code> that divides <code>fsdp_size * tp_size</code>) and reaches them through the implementation <code>ep_implementation</code> names, see <a href="#choosing-an-implementation">Choosing an implementation</a>.</p> <!--[2--><h3 class="relative group"><a id="choosing-an-implementation" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#choosing-an-implementation"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Choosing an implementation<!----></span></h3><!--]--><!----> <p><code>"ep_dispatch_experts"</code> shards the experts once and picks how tokens reach them from the implementations registered in <code>EPImplementationInterface</code>, selected with <code>ep_implementation</code> in <a href="/docs/transformers/pr_48566/en/expert_parallelism#transformers.DistributedConfig">DistributedConfig</a>:</p> <table><thead><tr><th align="left"><code>ep_implementation</code></th><th align="left">advantages</th><th align="left">disadvantages</th></tr></thead><tbody><tr><td align="left"><code>"all_to_all"</code>, see <a href="#token-dispatch">token dispatch</a></td><td align="left">Works for any layout, including <code>ep_size > tp_size</code>. Only the routed tokens move.</td><td align="left">Two all-to-all collectives and one host sync per layer, so it does not capture in a CUDA graph.</td></tr><tr><td align="left"><code>"all_reduce"</code></td><td align="left">No tokens move and no host sync, so decode captures in a CUDA graph.</td><td align="left">Requires <code>ep_size=tp_size</code>. Every rank runs its experts over the whole batch, so its cost grows with the batch.</td></tr><tr><td align="left"><code>"deepgemm_megamoe"</code></td><td align="left">DeepGEMM Mega MoE fuses the dispatch, the experts and the combine in one kernel. Every layer of an EP group shares its symmetric-memory buffers.</td><td align="left">Requires <code>ep_size=tp_size</code>, and FP8 activations with FP4 experts on SM100+. Inference only.</td></tr><tr><td align="left"><code>"auto"</code>, the default</td><td align="left">Runs <code>"all_reduce"</code> at inference when the ranks of an expert group share one batch, and <code>"all_to_all"</code> otherwise.</td><td align="left">Not always the fastest choice: a workload can benefit from a custom implementation, see <a href="#adding-an-implementation">Adding an implementation</a>.</td></tr></tbody></table> <blockquote class="note"><p><code>ep_implementation</code> only chooses within the parallel sizes set in <a href="/docs/transformers/pr_48566/en/expert_parallelism#transformers.DistributedConfig">DistributedConfig</a>. When the ranks of an expert group hold different batches, for example <code>tp_size=1</code> with <code>fsdp_size=ep_size</code>, <code>"all_to_all"</code> is the only implementation that applies.</p></blockquote> <!--[2--><h3 class="relative group"><a id="adding-an-implementation" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#adding-an-implementation"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Adding an implementation<!----></span></h3><!--]--><!----> <p>An implementation subclasses <code>EPImplementation</code> and registers under the name <code>ep_implementation</code> selects. It is built once per experts module, so it can keep state such as communication buffers. Calling it with the batch this rank holds, with global expert ids, returns the weighted expert output for each of its tokens. It runs this rank’s experts with <code>experts_forward</code>, on the rows, local expert ids and weights it brought them.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">from</span> transformers.distributed.expert_parallel <span class="hljs-keyword">import</span> EPImplementation, EPImplementationInterface | |
| <span class="hljs-keyword">class</span> <span class="hljs-title class_">MyEP</span>(<span class="hljs-title class_ inherited__">EPImplementation</span>): | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">__call__</span>(<span class="hljs-params">self, hidden_states, top_k_index, top_k_weights</span>): | |
| tokens, expert_ids, weights = ... <span class="hljs-comment"># bring this rank's experts the rows routed to them</span> | |
| expert_output = <span class="hljs-variable language_">self</span>.experts_forward(tokens, expert_ids, weights) | |
| <span class="hljs-keyword">return</span> ... <span class="hljs-comment"># the weighted expert output for each token of the batch this rank holds</span> | |
| EPImplementationInterface.register(<span class="hljs-string">"my_ep"</span>, MyEP) | |
| distributed_config = DistributedConfig(tp_size=<span class="hljs-number">1</span>, fsdp_size=<span class="hljs-number">8</span>, ep_size=<span class="hljs-number">8</span>, ep_implementation=<span class="hljs-string">"my_ep"</span>)<!----></pre></div><!----> <blockquote class="note"><p>Gradients must reach every rank that holds a token, routing weights included, so wrap a collective that autograd does not differentiate in a <code>torch.autograd.Function</code>.</p></blockquote> <blockquote class="tip"><p><code>enable_expert_parallel=True</code> without <code>ep_size</code> is deprecated and emits a <code>FutureWarning</code>. It shards only the experts, like <code>ep_size=tp_size, tp_plan={}</code>.</p></blockquote> <p>Launch your inference script with <a href="https://pytorch.org/docs/stable/elastic/run.html" rel="nofollow">torchrun</a>. The number of processes must equal <code>tp_size * fsdp_size * pp_size</code>, and <code>ep_size</code> must evenly divide the number of experts.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-zsh "><!---->torchrun --nproc-per-node 8 your_script.py<!----></pre></div><!----> <!--[2--><h3 class="relative group"><a id="overriding-the-plans" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#overriding-the-plans"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Overriding the plans<!----></span></h3><!--]--><!----> <p>Pass <code>tp_plan={...}</code> or <code>ep_plan={...}</code> to <a href="/docs/transformers/pr_48566/en/expert_parallelism#transformers.DistributedConfig">DistributedConfig</a> to override individual rules of the predefined plans. Unspecified rules are kept, and the merged plans are stored on the model. Each key must match a module, a parameter, or an existing plan entry; otherwise loading raises a <code>ValueError</code> before anything is sharded. Use the full path as seen from the loaded model, so <code>model.layers.*</code> for a causal LM and <code>layers.*</code> for its base model.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!---->distributed_config = DistributedConfig( | |
| tp_size=<span class="hljs-number">4</span>, | |
| ep_size=<span class="hljs-number">4</span>, | |
| tp_plan={<span class="hljs-string">"model.layers.*.self_attn.q_proj"</span>: <span class="hljs-string">"colwise_rep"</span>}, | |
| ep_plan={<span class="hljs-string">"model.layers.*.mlp.experts.down_proj"</span>: <span class="hljs-string">"grouped_gemm"</span>}, | |
| )<!----></pre></div><!----> <p>Providing a plan does not infer parallel sizes: set <code>tp_size</code> and <code>ep_size</code> explicitly.</p> <p>With <code>ep_size=tp_size</code>, the default <code>"auto"</code> implementation runs <code>"all_reduce"</code> at inference and dispatches tokens in training. To mask and all-reduce in training too, set <code>ep_implementation="all_reduce"</code>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!---->distributed_config = DistributedConfig(tp_size=<span class="hljs-number">4</span>, ep_size=<span class="hljs-number">4</span>, ep_implementation=<span class="hljs-string">"all_reduce"</span>)<!----></pre></div><!----> <!--[1--><h2 class="relative group"><a id="token-dispatch" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#token-dispatch"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Token dispatch<!----></span></h2><!--]--><!----> <p>Token dispatch is the <code>"all_to_all"</code> implementation: each rank trains on its own part of the batch. At every MoE layer, a rank routes its tokens, sends each (token, expert) pair to the rank that owns the expert with an all-to-all, runs its local experts on what it receives, gets the results back with a second all-to-all and combines them with the routing weights. Only the routed activations and expert outputs travel, and no rank computes experts for tokens it does not own.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM | |
| <span class="hljs-keyword">from</span> transformers.distributed <span class="hljs-keyword">import</span> DistributedConfig | |
| distributed_config = DistributedConfig( | |
| tp_size=<span class="hljs-number">1</span>, | |
| fsdp_size=<span class="hljs-number">8</span>, | |
| ep_size=<span class="hljs-number">4</span>, | |
| ) | |
| model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">"Qwen/Qwen3-30B-A3B"</span>, distributed_config=distributed_config)<!----></pre></div><!----> <p>With <code>tp_size=1</code>, <code>ep_size</code> must divide <code>fsdp_size</code> and the number of experts, and attention is not bound by <code>num_key_value_heads</code>. For the rest of the model:</p> <ul><li>The parameters outside the experts are sharded with <a href="./fsdp">FSDP2</a> across <code>fsdp</code>, which reduces their gradients.</li> <li>The experts are sharded across <code>ep</code> and, when <code>efsdp_size = fsdp_size * tp_size // ep_size</code> is larger than one, additionally FSDP-sharded across <code>efsdp</code>. They are always FSDP-wrapped, so <code>fsdp_mixed_precision</code> and <code>fsdp_cpu_offload</code> apply to them too and <a href="/docs/transformers/pr_48566/en/main_classes/model#transformers.PreTrainedModel.save_pretrained">save_pretrained()</a> gathers them like any other parameter.</li> <li>An expert parallel group holds <code>ep_size / tp_size</code> batches, so an expert’s gradient is a sum over that many batches. The <code>efsdp</code> reduction divides by <code>fsdp_size</code> instead of its group size, which gives the same per-batch average FSDP2 takes for the dense modules.</li> <li>Every local expert also processes one zero pad row per layer. A rank whose experts received no tokens still joins the reverse all-to-all and the expert gradient reduction.</li></ul> <!--[2--><h3 class="relative group"><a id="how-the-sizes-combine" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#how-the-sizes-combine"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->How the sizes combine<!----></span></h3><!--]--><!----> <ul><li><code>tp_size * fsdp_size</code> is the number of processes. <code>ep_size</code> adds none: it regroups the same ranks for the expert weights only.</li> <li><code>ep_size</code> cuts the expert list into <code>ep_size</code> blocks. Each rank computes <code>num_experts / ep_size</code> experts, and <code>ep_size</code> consecutive ranks hold one complete set. That set of ranks is the group the all-to-all runs in.</li> <li><code>efsdp_size = fsdp_size * tp_size / ep_size</code> is how many complete copies of the expert set exist. Ranks at the same position in different copies shard those experts for memory and average their gradients, like FSDP does for the dense modules.</li> <li>The batch a rank holds depends on <code>fsdp</code> only. Consecutive ranks form a TP group and get the same batch; the <code>fsdp_size</code> groups get different batches.</li></ul> <p>Eight processes, <code>tp_size=2, fsdp_size=4</code>, eight experts:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-text "><!---->rank 0 1 2 3 4 5 6 7 | |
| batch [====B0====] [====B1====] [====B2====] [====B3====] one batch per TP pair | |
| tp 0 1 0 1 0 1 0 1 | |
| ep_size=2 E0-3 E4-7 E0-3 E4-7 E0-3 E4-7 E0-3 E4-7 group = a TP pair, efsdp_size=4 | |
| ep_size=4 E0E1 E2E3 E4E5 E6E7 E0E1 E2E3 E4E5 E6E7 group = two pairs, efsdp_size=2 | |
| ep_size=8 E0 E1 E2 E3 E4 E5 E6 E7 group = all ranks, efsdp_size=1<!----></pre></div><!----> <p>Two numbers follow from the picture:</p> <ul><li>Inside an EP group, each token exists <code>tp_size</code> times, once per rank of the pair that holds its batch. This does not depend on <code>ep_size</code>.</li> <li>An EP group holds <code>ep_size / tp_size</code> different batches. This is the count an expert’s gradient sums over.</li></ul> <!--[2--><h3 class="relative group"><a id="with-tensor-parallelism" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#with-tensor-parallelism"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->With tensor parallelism<!----></span></h3><!--]--><!----> <p>Set <code>tp_size > 1</code> to shard the dense modules with the TP plan while the experts use dispatch. On eight processes:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!---->distributed_config = DistributedConfig( | |
| tp_size=<span class="hljs-number">2</span>, | |
| fsdp_size=<span class="hljs-number">4</span>, | |
| ep_size=<span class="hljs-number">4</span>, | |
| )<!----></pre></div><!----> <p>Each pair of TP ranks receives the same batch, because tensor parallelism replicates the activations inside the pair. If both ranks dispatched all of their tokens, the rank owning an expert would receive every token twice, compute it twice, and its weight gradient would double. Experts are whole on one rank, so the duplicate cannot be split by weights, and the owner is usually another rank, so it cannot be resolved by ownership as masking does. The pair therefore splits the rows: each TP rank dispatches a disjoint <code>1 / tp_size</code> of the tokens, results come back to the rank that sent them, and an all-reduce over the pair of the zero-padded halves restores the replicated output the next layer expects. The split is by <code>tp_size</code>, not <code>ep_size</code>, since only the ranks that hold a batch can send it. Expert groups span four ranks and each expert is FSDP-sharded across <code>efsdp_size = 2</code> ranks, while the trunk’s FSDP group spans four ranks. The model’s usual TP constraints, such as attention-head divisibility, still apply to the dense modules. Token slices may be uneven or empty, including during single-token decoding.</p> <p>Token dispatch cannot be combined yet with pipeline parallelism yet; use <code>pp_size=1</code> (not tested yet) | |
| These configurations each use eight GPUs:</p> <table><thead><tr><th align="left">Configuration</th><th align="left">Result</th></tr></thead><tbody><tr><td align="left"><code>DistributedConfig(tp_size=4, fsdp_size=2, ep_size=4)</code></td><td align="left">Each TP group of four shares one batch. Training dispatches tokens, each rank taking a quarter of the batch; inference uses <code>"all_reduce"</code>; setting <code>ep_implementation="all_reduce"</code> uses it throughout.</td></tr><tr><td align="left"><code>DistributedConfig(tp_size=1, fsdp_size=8, ep_size=4)</code></td><td align="left">Dispatch with an independent batch on each rank, no slicing, and experts FSDP-sharded across pairs of ranks.</td></tr><tr><td align="left"><code>DistributedConfig(tp_size=2, fsdp_size=4, ep_size=4)</code></td><td align="left">Dispatch with a TP pair per batch, each pair slicing its batch in two; two batches per expert group.</td></tr><tr><td align="left"><code>DistributedConfig(tp_size=8, ep_size=8)</code></td><td align="left">Every rank shares one batch. Training dispatches tokens, each rank taking an eighth of the batch; inference uses <code>"all_reduce"</code>; setting <code>ep_implementation="all_reduce"</code> uses it throughout.</td></tr></tbody></table> <blockquote class="warning"><p>The <a href="/docs/transformers/pr_48566/en/main_classes/trainer#transformers.Trainer">Trainer</a> does not account for token dispatch yet: batch and token counting assume the all-reduce layout, where the ranks of a TP group share a batch and <code>fsdp_size</code> data-parallel shards exist. Trainer support for dispatch comes in a follow-up.</p></blockquote> <!--[1--><h2 class="relative group"><a id="combining-with-fsdp2" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#combining-with-fsdp2"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Combining with FSDP2<!----></span></h2><!--]--><!----> <p>Tensor and expert parallelism shard the weights across their own meshes, <code>tp</code> and <code>ep</code>, but the optimizer state and the modules without a rule are still replicated on every rank of the group, which limits how large a model you can train. Add <a href="./fsdp">FSDP2</a> on another mesh dimension with <code>fsdp_size</code>. With masking and all-reduce, keep <code>ep_size=tp_size</code> and pass <code>ep_implementation="all_reduce"</code>.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM | |
| <span class="hljs-keyword">from</span> transformers.distributed <span class="hljs-keyword">import</span> DistributedConfig | |
| distributed_config = DistributedConfig( | |
| tp_size=<span class="hljs-number">4</span>, | |
| ep_size=<span class="hljs-number">4</span>, <span class="hljs-comment"># expert parallel size, must match tp_size with masking and all-reduce</span> | |
| fsdp_size=<span class="hljs-number">2</span>, <span class="hljs-comment"># data parallel shards</span> | |
| ep_implementation=<span class="hljs-string">"all_reduce"</span>, <span class="hljs-comment"># "auto" would dispatch tokens in training, which the Trainer does not support yet</span> | |
| ) | |
| model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">"Qwen/Qwen3-30B-A3B"</span>, distributed_config=distributed_config)<!----></pre></div><!----> <p>Load the model as usual, then train with <a href="/docs/transformers/pr_48566/en/main_classes/trainer#transformers.Trainer">Trainer</a>. It takes the gradient norm across both meshes and gives each mesh its own optimizer param group. <a href="/docs/transformers/pr_48566/en/main_classes/trainer#transformers.Trainer.save_model">save_model()</a> gathers sharded weights into a regular checkpoint. This requires <code>accelerate>=1.12</code> so the <code>Trainer</code> can mirror <code>tp_size</code> and <code>fsdp_size</code> into <code>~Accelerate.ParallelismConfig</code>.</p> <p>The table below compares EP-only training with 2D EP+FSDP2 on 8xH100 GPUs. The workload is full fine-tuning of Qwen3-30B-A3B in bf16 at sequence length 2048. More FSDP shards cut peak memory, and tokens/s drop some because FSDP2 all-gathers and reduce-scatters the experts across <code>fsdp</code>.</p> <table><thead><tr><th>configuration</th><th>tokens/s/GPU</th><th>peak memory/GPU</th></tr></thead><tbody><tr><td><code>tp_size=8</code></td><td>3485</td><td>38.6 GB</td></tr><tr><td><code>tp_size=4, fsdp_size=2</code></td><td>2900</td><td>34.2 GB</td></tr><tr><td><code>tp_size=2, fsdp_size=4</code></td><td>2830</td><td>32.3 GB</td></tr></tbody></table> <blockquote class="warning"><p>Resuming from a checkpoint is not supported yet for models sharded at load time, so the <a href="/docs/transformers/pr_48566/en/main_classes/trainer#transformers.Trainer">Trainer</a> only accepts <code>save_only_model=True</code> or <code>save_strategy="no"</code> for them.</p></blockquote> <!--[1--><h2 class="relative group"><a id="transformers.DistributedConfig" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#transformers.DistributedConfig"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->API reference<!----></span></h2><!--]--><!----> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><div><span class="group flex space-x-1.5 items-center text-gray-800 bg-gradient-to-r rounded-tr-lg -mt-4 -ml-4 pt-3 px-2.5" id="transformers.DistributedConfig"><!----><h3 class="!m-0"><span class="flex-1 break-all md:text-lg bg-gradient-to-r px-2.5 py-1.5 rounded-xl from-indigo-50/70 to-white dark:from-gray-900 dark:to-gray-950 dark:text-indigo-300 text-indigo-700"><svg class="mr-1.5 text-indigo-500 inline-block -mt-0.5" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width=".8em" height=".8em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 24 24"><path class="uim-quaternary" d="M20.23 7.24L12 12L3.77 7.24a1.98 1.98 0 0 1 .7-.71L11 2.76c.62-.35 1.38-.35 2 0l6.53 3.77c.29.173.531.418.7.71z" opacity=".25" fill="currentColor"></path><path class="uim-tertiary" d="M12 12v9.5a2.09 2.09 0 0 1-.91-.21L4.5 17.48a2.003 2.003 0 0 1-1-1.73v-7.5a2.06 2.06 0 0 1 .27-1.01L12 12z" opacity=".5" fill="currentColor"></path><path class="uim-primary" d="M20.5 8.25v7.5a2.003 2.003 0 0 1-1 1.73l-6.62 3.82c-.275.13-.576.198-.88.2V12l8.23-4.76c.175.308.268.656.27 1.01z" fill="currentColor"></path></svg><span class="font-light">class</span> <span class="font-medium">transformers.</span><span class="font-semibold">DistributedConfig</span></span></h3><!----> <a id="transformers.DistributedConfig" class="header-link invisible with-hover:group-hover:visible pr-2" href="#transformers.DistributedConfig"><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></a> <!--[0--><a class="!ml-auto !text-gray-400 !no-underline text-sm flex items-center" href="https://github.com/huggingface/transformers/blob/vr_48566/src/transformers/distributed/configuration_utils.py#L26" target="_blank"><span><</span> <span class="hidden md:block mx-0.5 hover:!underline">source</span> <span>></span></a><!--]--></span> <!--[0--><p class="font-mono text-xs md:text-sm !leading-relaxed !my-6"><span>(</span> <!--[--><span class="comma cursor-pointer"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">tp_size<span class="opacity-60">: int | None = None</span></span></span><span class="comma cursor-pointer"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">tp_plan<span class="opacity-60">: typing.Union[dict[str, str], typing.Literal['auto'], NoneType] = None</span></span></span><span class="comma cursor-pointer"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">enable_sequence_parallel<span class="opacity-60">: bool = False</span></span></span><span class="comma cursor-pointer"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">enable_expert_parallel<span class="opacity-60">: bool = False</span></span></span><span class="comma cursor-pointer"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">fsdp_size<span class="opacity-60">: int | None = None</span></span></span><span class="comma cursor-pointer"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">dp_replicate_size<span class="opacity-60">: int | None = None</span></span></span><span class="comma cursor-pointer"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">fsdp_plan<span class="opacity-60">: dict[str, str] | None = None</span></span></span><span class="comma cursor-pointer"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">fsdp_cpu_offload<span class="opacity-60">: bool = False</span></span></span><span class="comma cursor-pointer"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">fsdp_mixed_precision<span class="opacity-60">: bool = False</span></span></span><span class="comma cursor-pointer"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">pp_size<span class="opacity-60">: int | None = None</span></span></span><span class="comma cursor-pointer"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">ep_size<span class="opacity-60">: int | None = None</span></span></span><span class="comma cursor-pointer"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">ep_plan<span class="opacity-60">: dict[str, str] | None = None</span></span></span><span class="comma cursor-pointer"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">ep_implementation<span class="opacity-60">: str = 'auto'</span></span></span><!--]--> <span>)</span> <!--[-1--><!--]--></p><!--]--> <div class="!mb-10 relative docstring-details "><!--[-1--><!--]--> <!--[0--><p class="flex items-center font-semibold !mt-2 !mb-2 text-gray-800">Parameters <span class="flex-auto border-t-2 border-gray-100 dark:border-gray-700 ml-3"></span></p> <ul class="px-2"><!--[--><li class="text-base !pl-4 my-3 rounded "><span class="group flex space-x-1.5 items-start"><a id="transformers.DistributedConfig.tp_size" class="header-link block pr-0.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#transformers.DistributedConfig.tp_size"><span><svg class="text-smd" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!----><strong>tp_size</strong> (<code>int</code>, <em>optional</em>) — | |
| Number of devices for tensor parallelism. If <code>None</code> and <code>tp_plan</code> is set, defaults to | |
| <code>WORLD_SIZE // (other_parallel_size)</code>. If <code>None</code> and no <code>tp_plan</code> is set, defaults to 1.<!----></span></span></li><li class="text-base !pl-4 my-3 rounded "><span class="group flex space-x-1.5 items-start"><a id="transformers.DistributedConfig.tp_plan" class="header-link block pr-0.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#transformers.DistributedConfig.tp_plan"><span><svg class="text-smd" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!----><strong>tp_plan</strong> (<code>dict[str, str]</code> or <code>"auto"</code>, <em>optional</em>) — | |
| Tensor parallel sharding plan. Pass <code>"auto"</code>, or leave as <code>None</code> when <code>tp_size</code> is set, to use the | |
| model’s predefined <code>base_model_tp_plan</code>. Pass a dictionary to override individual rules of that plan; | |
| unspecified rules are kept. An empty dictionary applies no tensor parallel rule.<!----></span></span></li><li class="text-base !pl-4 my-3 rounded "><span class="group flex space-x-1.5 items-start"><a id="transformers.DistributedConfig.enable_sequence_parallel" class="header-link block pr-0.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#transformers.DistributedConfig.enable_sequence_parallel"><span><svg class="text-smd" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!----><strong>enable_sequence_parallel</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Reserved for sequence parallelism. Not wired up yet.<!----></span></span></li><li class="text-base !pl-4 my-3 rounded "><span class="group flex space-x-1.5 items-start"><a id="transformers.DistributedConfig.enable_expert_parallel" class="header-link block pr-0.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#transformers.DistributedConfig.enable_expert_parallel"><span><svg class="text-smd" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!----><strong>enable_expert_parallel</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Deprecated, removed in v5.21. Without <code>ep_size</code>, same as <code>ep_size=tp_size, tp_plan={}</code>: the | |
| expert-parallel plan alone across <code>tp_size</code> ranks. An explicit <code>ep_size</code> takes precedence. This flag | |
| does not change <code>tp_size</code> or <code>fsdp_size</code>.<!----></span></span></li><li class="text-base !pl-4 my-3 rounded "><span class="group flex space-x-1.5 items-start"><a id="transformers.DistributedConfig.fsdp_size" class="header-link block pr-0.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#transformers.DistributedConfig.fsdp_size"><span><svg class="text-smd" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!----><strong>fsdp_size</strong> (<code>int</code>, <em>optional</em>) — | |
| Number of devices for FSDP (data parallelism). If <code>None</code> and <code>tp_size</code> is set, defaults to 1.<!----></span></span></li><li class="text-base !pl-4 my-3 rounded "><span class="group flex space-x-1.5 items-start"><a id="transformers.DistributedConfig.dp_replicate_size" class="header-link block pr-0.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#transformers.DistributedConfig.dp_replicate_size"><span><svg class="text-smd" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!----><strong>dp_replicate_size</strong> (<code>int</code>, <em>optional</em>) — | |
| Number of FSDP replicas (HSDP). Parameters are sharded across <code>fsdp_size</code> devices and replicated | |
| <code>dp_replicate_size</code> times: gradients are reduce-scattered within a replica, then all-reduced across | |
| replicas. Defaults to 1. We suggest it for multi-node training: plain FSDP all-gathers parameters and | |
| reduce-scatters gradients across all devices at every step, so this traffic goes over the slower | |
| inter-node links. Set <code>dp_replicate_size</code> to the number of nodes and <code>fsdp_size * tp_size</code> to the number | |
| of devices per node: the sharding traffic then stays on the fast intra-node links, and only the gradient | |
| all-reduce crosses nodes. The cost is more memory per device, since parameters are sharded within a node.<!----></span></span></li><li class="text-base !pl-4 my-3 rounded "><span class="group flex space-x-1.5 items-start"><a id="transformers.DistributedConfig.fsdp_plan" class="header-link block pr-0.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#transformers.DistributedConfig.fsdp_plan"><span><svg class="text-smd" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!----><strong>fsdp_plan</strong> (<code>dict[str, str]</code>, <em>optional</em>) — | |
| FSDP2 sharding plan. Leave as <code>None</code> to use the model’s predefined <code>base_model_fsdp_plan</code> and | |
| class-level <code>_fsdp_plan</code>. Pass a dictionary to override individual rules; unspecified rules are kept. | |
| Applied only when <code>dp_replicate_size * fsdp_size > 1</code>.<!----></span></span></li><li class="text-base !pl-4 my-3 rounded "><span class="group flex space-x-1.5 items-start"><a id="transformers.DistributedConfig.fsdp_cpu_offload" class="header-link block pr-0.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#transformers.DistributedConfig.fsdp_cpu_offload"><span><svg class="text-smd" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!----><strong>fsdp_cpu_offload</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether to enable CPU offloading for FSDP2.<!----></span></span></li><li class="text-base !pl-4 my-3 rounded "><span class="group flex space-x-1.5 items-start"><a id="transformers.DistributedConfig.fsdp_mixed_precision" class="header-link block pr-0.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#transformers.DistributedConfig.fsdp_mixed_precision"><span><svg class="text-smd" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!----><strong>fsdp_mixed_precision</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether to enable mixed precision for FSDP2.<!----></span></span></li><li class="text-base !pl-4 my-3 rounded "><span class="group flex space-x-1.5 items-start"><a id="transformers.DistributedConfig.pp_size" class="header-link block pr-0.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#transformers.DistributedConfig.pp_size"><span><svg class="text-smd" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!----><strong>pp_size</strong> (<code>int</code>, <em>optional</em>) — | |
| Number of devices for pipeline parallelism. If <code>None</code> and another parallel mode is set, defaults to 1.<!----></span></span></li><li class="text-base !pl-4 my-3 rounded "><span class="group flex space-x-1.5 items-start"><a id="transformers.DistributedConfig.ep_size" class="header-link block pr-0.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#transformers.DistributedConfig.ep_size"><span><svg class="text-smd" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!----><strong>ep_size</strong> (<code>int</code>, <em>optional</em>) — | |
| Number of devices owning distinct expert shards. Defaults to 1. Set it explicitly to enable EP. Must be | |
| a multiple of <code>tp_size</code> and divide <code>fsdp_size * tp_size</code>. <code>ep_size > tp_size</code> needs an | |
| <code>"ep_dispatch_experts"</code> rule with an <code>ep_implementation</code> that exchanges tokens, such as <code>"all_to_all"</code>.<!----></span></span></li><li class="text-base !pl-4 my-3 rounded "><span class="group flex space-x-1.5 items-start"><a id="transformers.DistributedConfig.ep_plan" class="header-link block pr-0.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#transformers.DistributedConfig.ep_plan"><span><svg class="text-smd" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!----><strong>ep_plan</strong> (<code>dict[str, str]</code>, <em>optional</em>) — | |
| Expert parallel sharding plan. Leave as <code>None</code> to use the model’s predefined <code>base_model_ep_plan</code>. Pass a | |
| dictionary to override individual rules of that plan; unspecified rules are kept. Applied only when | |
| <code>ep_size > 1</code>, and its rules take precedence over <code>tp_plan</code> rules for the same modules. An | |
| <code>"ep_dispatch_experts"</code> rule lets <code>ep_implementation</code> pick how tokens reach the experts.<!----></span></span></li><li class="text-base !pl-4 my-3 rounded "><span class="group flex space-x-1.5 items-start"><a id="transformers.DistributedConfig.ep_implementation" class="header-link block pr-0.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#transformers.DistributedConfig.ep_implementation"><span><svg class="text-smd" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!----><strong>ep_implementation</strong> (<code>str</code>, <em>optional</em>, defaults to <code>"auto"</code>) — | |
| How an <code>"ep_dispatch_experts"</code> rule brings the experts their tokens. It names an <code>EPImplementation</code> | |
| registered in <code>EPImplementationInterface</code>: | |
| <ul> | |
| <li><code>"all_to_all"</code> exchanges the routed tokens with all-to-all collectives.</li> | |
| <li><code>"all_reduce"</code> runs each rank’s experts on the batch the EP group shares, and sums the outputs. It | |
| requires <code>ep_size == tp_size</code>.</li> | |
| <li><code>"deepgemm_megamoe"</code> hands that batch to DeepGEMM Mega MoE, whose kernel exchanges the tokens itself. | |
| Loading with <code>experts_implementation="deepgemm_megamoe"</code> selects it.</li> | |
| <li><code>"auto"</code> runs <code>"all_reduce"</code> at inference when the EP group shares one batch, and <code>"all_to_all"</code> | |
| otherwise.</li> | |
| </ul><!----></span></span></li><!--]--></ul><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--></div></div><!----> <p>Configuration for native distributed inference and training with tensor, pipeline, or FSDP2 parallelism.</p></div> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/transformers/blob/main/docs/source/en/expert_parallelism.md" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg><!----> <span><span class="underline">Update</span> on GitHub</span></a><!----> <p></p><!--]--><!----><!--]--><!--]--><!--]--> <!--[-1--><!--]--><!--]--> | |
| <script> | |
| { | |
| __sveltekit_1e8t8or = { | |
| base: "/docs/transformers/pr_48566/en", | |
| assets: "/docs/transformers/pr_48566/en" | |
| }; | |
| const element = document.currentScript.parentElement; | |
| Promise.all([ | |
| import("/docs/transformers/pr_48566/en/_app/immutable/entry/start.GdwabqXz.js"), | |
| import("/docs/transformers/pr_48566/en/_app/immutable/entry/app.CrR-wy65.js") | |
| ]).then(([kit, app]) => { | |
| kit.start(app, element, { | |
| node_ids: [0, 2], | |
| data: [null,null], | |
| form: null, | |
| error: null | |
| }); | |
| }); | |
| } | |
| </script> | |
Xet Storage Details
- Size:
- 77.5 kB
- Xet hash:
- 0cacb3486ac462a44c6531e4f702bf79c0d61be9710da203a235bf0651ded425
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.