Buckets:

download
raw
11.6 kB
import"../chunks/DsnmJJEf.js";import{i as b,h as f,C as j,H as s,a,E as Z,s as I}from"../chunks/DBNuz8hP.js";import{p as E,o as B,s as e,f as G,a as M,b as v,d as T,n as k}from"../chunks/CQ2f15a1.js";const W='{"title":"AWS Trainium","local":"aws-trainium","sections":[{"title":"Eager mode","local":"eager-mode","sections":[],"depth":2},{"title":"Compiled mode","local":"compiled-mode","sections":[],"depth":2},{"title":"Tensor parallelism","local":"tensor-parallelism","sections":[],"depth":2},{"title":"Native FLUX.2 Klein serving proof of concept","local":"native-flux2-klein-serving-proof-of-concept","sections":[],"depth":2}],"depth":1}';var X=T('<meta name="hf:doc:metadata"/>'),U=T(`<p></p> <!> <!> <p><a href="https://github.com/aws-neuron/torch-neuronx" rel="nofollow">TorchNeuron</a> is the native PyTorch backend for Trainium2 and Trainium3. Diffusers runs on these instances through the <code>torch_neuronx</code> package, which registers a <code>"neuron"</code> device, so pipelines run with the same code as on any other device.</p> <p>Two execution modes are available:</p> <table><thead><tr><th>Mode</th><th>How to activate</th><th>Notes</th></tr></thead><tbody><tr><td>Eager</td><td><code>pipe.to("neuron")</code></td><td>Operations dispatched one at a time, asynchronous</td></tr><tr><td>Compile</td><td><code>torch.compile(module, backend="neuron")</code></td><td>Whole-graph compilation, static shapes</td></tr></tbody></table> <p>Follow the <a href="https://awsdocs-neuron.readthedocs-hosted.com/en/latest/" rel="nofollow">AWS Neuron documentation</a> to set up the Neuron
SDK and install <code>torch_neuronx</code>. Once installed, <code>import torch</code> loads it automatically and registers the <code>"neuron"</code> device, so <code>pipe.to("neuron")</code> is the only change needed. Add <code>import torch_neuronx</code> only if you disabled backend
autoloading with <code>TORCH_DEVICE_BACKEND_AUTOLOAD=0</code>.</p> <p>In eager mode, each operation compiles the first time it runs. In compiled mode, the first call compiles the graph. Both modes cache the compiled Neuron executable format (NEFF) under <code>tmp/neff_cache</code> by default, and that directory is cleared on reboot. Set <code>TORCH_NEURONX_NEFF_CACHE_DIR</code> to a persistent directory to keep the cache.</p> <!> <p>FLUX.2-klein-4B fits on a single NeuronCore, so move the whole pipeline to the <code>"neuron"</code> device.</p> <!> <p>If a model is too large for a single NeuronCore, or you have several cores and want lower latency, shard the models
across cores instead. See the <a href="#tensor-parallelism">Tensor parallelism</a> section.</p> <!> <p>TorchNeuron registers <code>"neuron"</code> as a <code>torch.compile</code> backend name, so components compile like any other <code>torch.compile</code> target. The first call (warmup) is slow because it compiles; later calls with the same shapes reuse the compiled graph.</p> <blockquote class="important"><p>TorchNeuron requires <em>static shapes</em>, so pass <code>dynamic=False</code>. A new <code>height</code> or <code>width</code> compiles again for that shape. Shapes already seen are reused. Changing <code>num_inference_steps</code> doesn’t recompile.</p></blockquote> <!> <p>For transformers with repeated blocks, use <code>compile_repeated_blocks(...)</code> to compile those blocks individually. This block is reused and reduces compile time considerably.</p> <!> <p>Follow the <a href="./tensor_parallelism">Tensor parallelism</a> guide for model loading,
sharding plans, and limitations. To adapt its example for Neuron, replace the CUDA process-group and device setup
with the following.</p> <p>On Neuron, initialize the process group with <code>backend="neuron"</code> and build the mesh with <code>DeviceMesh("neuron", ...)</code>.</p> <!> <p>Pass this <code>parallel_config</code> to the transformer’s <a href="/docs/diffusers/pr_15249/en/api/models/overview#diffusers.ModelMixin.from_pretrained">from_pretrained()</a> and use <code>device</code> for the remaining
pipeline components.</p> <p>Launch one process per NeuronCore. Set <code>--nproc_per_node</code> to the number of NeuronCores you want to shard across.</p> <!> <!> <p>The experimental <code>add-diffusers-support</code> branch of <code>vllm-omni-neuron</code> wraps <code>Flux2KleinPipeline</code> with vLLM-Omni’s Diffusers adapter for single-device BF16
inference. This is separate from the Optimum Neuron export workflow above.</p> <p>Diffusers owns model execution; the plugin owns worker initialization, placement,
and compilation policy. The pinned serving runtime compiles the text encoder,
transformer, VAE decoder, scheduler step, and tensor preparation regions.
See that branch’s <code>examples/diffusers/README.md</code> for setup and serving examples.</p> <p>The real <code>black-forest-labs/FLUX.2-klein-4B</code> checkpoint has completed offline Omni
generation on Trainium2 at 256×256, four steps, BF16, and TP=1. HTTP serving,
larger shapes, and tensor parallelism remain unvalidated; this is not production
support.</p> <!> <p></p>`,1);function V(g,J){E(J,!1),B(()=>{new URLSearchParams(window.location.search).get("fw")}),b();var l=U();f("11xqcdu",u=>{var w=X();I(w,"content",W),M(u,w)});var o=e(G(l),2);j(o,{containerStyle:"float: inline-end; margin-inline-start: 10px; display: inline-flex; position: relative; z-index: 10;"});var n=e(o,2);s(n,{title:"AWS Trainium",local:"aws-trainium",headingTag:"h1"});var t=e(n,12);s(t,{title:"Eager mode",local:"eager-mode",headingTag:"h2"});var r=e(t,4);a(r,{code:"aW1wb3J0JTIwdG9yY2glMEElMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwRmx1eDJLbGVpblBpcGVsaW5lJTBBJTBBcGlwZSUyMCUzRCUyMEZsdXgyS2xlaW5QaXBlbGluZS5mcm9tX3ByZXRyYWluZWQoJTIyYmxhY2stZm9yZXN0LWxhYnMlMkZGTFVYLjIta2xlaW4tNEIlMjIlMkMlMjBkdHlwZSUzRHRvcmNoLmJmbG9hdDE2KSUwQXBpcGUudG8oJTIybmV1cm9uJTIyKSUwQSUwQWltYWdlJTIwJTNEJTIwcGlwZSglMEElMjAlMjAlMjAlMjBwcm9tcHQlM0QlMjJhJTIwZ29sZGVuJTIwcmV0cmlldmVyJTIwc3VyZmluZyUyMGElMjB3YXZlJTJDJTIwcGhvdG9yZWFsaXN0aWMlMjIlMkMlMEElMjAlMjAlMjAlMjBoZWlnaHQlM0QxMDI0JTJDJTBBJTIwJTIwJTIwJTIwd2lkdGglM0QxMDI0JTJDJTBBJTIwJTIwJTIwJTIwbnVtX2luZmVyZW5jZV9zdGVwcyUzRDQlMkMlMEEpLmltYWdlcyU1QjAlNUQlMEElMEFpbWFnZS5zYXZlKCUyMm91dHB1dC5wbmclMjIp",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Flux2KleinPipeline
pipe = Flux2KleinPipeline.from_pretrained(<span class="hljs-string">&quot;black-forest-labs/FLUX.2-klein-4B&quot;</span>, dtype=torch.bfloat16)
pipe.to(<span class="hljs-string">&quot;neuron&quot;</span>)
image = pipe(
prompt=<span class="hljs-string">&quot;a golden retriever surfing a wave, photorealistic&quot;</span>,
height=<span class="hljs-number">1024</span>,
width=<span class="hljs-number">1024</span>,
num_inference_steps=<span class="hljs-number">4</span>,
).images[<span class="hljs-number">0</span>]
image.save(<span class="hljs-string">&quot;output.png&quot;</span>)`,lang:"python",wrap:!1});var i=e(r,4);s(i,{title:"Compiled mode",local:"compiled-mode",headingTag:"h2"});var c=e(i,6);a(c,{code:"aW1wb3J0JTIwdG9yY2glMEElMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwRmx1eDJLbGVpblBpcGVsaW5lJTBBJTBBcGlwZSUyMCUzRCUyMEZsdXgyS2xlaW5QaXBlbGluZS5mcm9tX3ByZXRyYWluZWQoJTIyYmxhY2stZm9yZXN0LWxhYnMlMkZGTFVYLjIta2xlaW4tNEIlMjIlMkMlMjBkdHlwZSUzRHRvcmNoLmJmbG9hdDE2KSUwQXBpcGUudG8oJTIybmV1cm9uJTIyKSUwQXBpcGUudHJhbnNmb3JtZXIuY29tcGlsZShiYWNrZW5kJTNEJTIybmV1cm9uJTIyJTJDJTIwZnVsbGdyYXBoJTNEVHJ1ZSUyQyUyMGR5bmFtaWMlM0RGYWxzZSklMEElMEElMjMlMjBXYXJtdXAlMjAlRTIlODAlOTQlMjB0cmlnZ2VycyUyMHN0YXRpYyUyMGdyYXBoJTIwY29tcGlsYXRpb24uJTBBcGlwZShwcm9tcHQlM0QlMjJ3YXJtdXAlMjIlMkMlMjBoZWlnaHQlM0QxMDI0JTJDJTIwd2lkdGglM0QxMDI0JTJDJTIwbnVtX2luZmVyZW5jZV9zdGVwcyUzRDQpJTBBJTBBJTIzJTIwTGF0ZXIlMjBjYWxscyUyMHdpdGglMjB0aGUlMjBzYW1lJTIwc2hhcGVzJTIwcmV1c2UlMjB0aGUlMjBjb21waWxlZCUyMGdyYXBoLiUwQWltYWdlJTIwJTNEJTIwcGlwZSglMEElMjAlMjAlMjAlMjBwcm9tcHQlM0QlMjJhJTIwZ29sZGVuJTIwcmV0cmlldmVyJTIwc3VyZmluZyUyMGElMjB3YXZlJTJDJTIwcGhvdG9yZWFsaXN0aWMlMjIlMkMlMEElMjAlMjAlMjAlMjBoZWlnaHQlM0QxMDI0JTJDJTBBJTIwJTIwJTIwJTIwd2lkdGglM0QxMDI0JTJDJTBBJTIwJTIwJTIwJTIwbnVtX2luZmVyZW5jZV9zdGVwcyUzRDQlMkMlMEEpLmltYWdlcyU1QjAlNUQlMEElMEFpbWFnZS5zYXZlKCUyMm91dHB1dC5wbmclMjIp",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Flux2KleinPipeline
pipe = Flux2KleinPipeline.from_pretrained(<span class="hljs-string">&quot;black-forest-labs/FLUX.2-klein-4B&quot;</span>, dtype=torch.bfloat16)
pipe.to(<span class="hljs-string">&quot;neuron&quot;</span>)
pipe.transformer.<span class="hljs-built_in">compile</span>(backend=<span class="hljs-string">&quot;neuron&quot;</span>, fullgraph=<span class="hljs-literal">True</span>, dynamic=<span class="hljs-literal">False</span>)
<span class="hljs-comment"># Warmup — triggers static graph compilation.</span>
pipe(prompt=<span class="hljs-string">&quot;warmup&quot;</span>, height=<span class="hljs-number">1024</span>, width=<span class="hljs-number">1024</span>, num_inference_steps=<span class="hljs-number">4</span>)
<span class="hljs-comment"># Later calls with the same shapes reuse the compiled graph.</span>
image = pipe(
prompt=<span class="hljs-string">&quot;a golden retriever surfing a wave, photorealistic&quot;</span>,
height=<span class="hljs-number">1024</span>,
width=<span class="hljs-number">1024</span>,
num_inference_steps=<span class="hljs-number">4</span>,
).images[<span class="hljs-number">0</span>]
image.save(<span class="hljs-string">&quot;output.png&quot;</span>)`,lang:"python",wrap:!1});var p=e(c,4);s(p,{title:"Tensor parallelism",local:"tensor-parallelism",headingTag:"h2"});var d=e(p,6);a(d,{code:"aW1wb3J0JTIwdG9yY2glMEFpbXBvcnQlMjB0b3JjaC5kaXN0cmlidXRlZCUyMGFzJTIwZGlzdCUwQWZyb20lMjB0b3JjaC5kaXN0cmlidXRlZC5kZXZpY2VfbWVzaCUyMGltcG9ydCUyMERldmljZU1lc2glMEElMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwVGVuc29yUGFyYWxsZWxDb25maWclMEElMEFkaXN0LmluaXRfcHJvY2Vzc19ncm91cChiYWNrZW5kJTNEJTIybmV1cm9uJTIyKSUwQW1lc2glMjAlM0QlMjBEZXZpY2VNZXNoKCUyMm5ldXJvbiUyMiUyQyUyMGxpc3QocmFuZ2UoZGlzdC5nZXRfd29ybGRfc2l6ZSgpKSkpJTBBZGV2aWNlJTIwJTNEJTIwdG9yY2gubmV1cm9uLmN1cnJlbnRfZGV2aWNlKCklMEFwYXJhbGxlbF9jb25maWclMjAlM0QlMjBUZW5zb3JQYXJhbGxlbENvbmZpZyhtZXNoJTNEbWVzaCk=",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">import</span> torch.distributed <span class="hljs-keyword">as</span> dist
<span class="hljs-keyword">from</span> torch.distributed.device_mesh <span class="hljs-keyword">import</span> DeviceMesh
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> TensorParallelConfig
dist.init_process_group(backend=<span class="hljs-string">&quot;neuron&quot;</span>)
mesh = DeviceMesh(<span class="hljs-string">&quot;neuron&quot;</span>, <span class="hljs-built_in">list</span>(<span class="hljs-built_in">range</span>(dist.get_world_size())))
device = torch.neuron.current_device()
parallel_config = TensorParallelConfig(mesh=mesh)`,lang:"python",wrap:!1});var h=e(d,6);a(h,{code:"dG9yY2hydW4lMjAtLW5wcm9jX3Blcl9ub2RlJTNEOCUyMHRlbnNvcl9wYXJhbGxlbF9uZXVyb24ucHk=",highlighted:"torchrun --nproc_per_node=8 tensor_parallel_neuron.py",lang:"bash",wrap:!1});var m=e(h,2);s(m,{title:"Native FLUX.2 Klein serving proof of concept",local:"native-flux2-klein-serving-proof-of-concept",headingTag:"h2"});var y=e(m,8);Z(y,{source:"https://github.com/huggingface/diffusers/blob/main/docs/source/en/optimization/neuron.md"}),k(2),M(g,l),v()}export{V as component};

Xet Storage Details

Size:
11.6 kB
·
Xet hash:
2a7d58babc24f628a50f066a0e784e0055be1107345fb45b1118936600b90275

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.