Buckets:

hf-doc-build
/

doc

Files
xet
hf-doc-build/doc / diffusers /main /ar /optimization /para_attn.html
mishig
3 days ago
download
raw
57.6 kB
<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;ParaAttention&quot;,&quot;local&quot;:&quot;paraattention&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;ذاكرة التخزين المؤقت للكتلة الأولى&quot;,&quot;local&quot;:&quot;first-block-cache&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;تكميم fp8&quot;,&quot;local&quot;:&quot;fp8-quantization&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;توازي السياق&quot;,&quot;local&quot;:&quot;context-parallelism&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;المعايير&quot;,&quot;local&quot;:&quot;benchmarks&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}"/>
		<link href="/docs/diffusers/main/en/_app/immutable/entry/start.0S0_iEoG.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/I9eooN77.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/Bupkjr9f.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/entry/app.CPdOxCbw.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/CyvZ3FS5.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/l0QnnOED.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/DnVIJfC5.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/DsnmJJEf.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/nodes/0.DteTvmQZ.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/CoSrdQax.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/nodes/344.CCPw-O4m.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/DSPDl1gz.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/D5eijdnr.js" rel="modulepreload">
		<!--fka9nn--><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;ParaAttention&quot;,&quot;local&quot;:&quot;paraattention&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;First Block Cache&quot;,&quot;local&quot;:&quot;first-block-cache&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;fp8 quantization&quot;,&quot;local&quot;:&quot;fp8-quantization&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Context Parallelism&quot;,&quot;local&quot;:&quot;context-parallelism&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Benchmarks&quot;,&quot;local&quot;:&quot;benchmarks&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}"/><!---->
		<link href="/docs/diffusers/main/en/_app/immutable/assets/0.tn0RQdqM.css" rel="modulepreload"> <!--[--><!--[0--><!--[--><!--[0--><!--[--><p></p> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: inline-end; margin-inline-start: 10px; display: inline-flex; position: relative; z-index: 10;"><div dir="ltr" class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 h-7 max-sm:h-7 px-2 max-sm:px-1.5 text-sm font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0 hover:text-gray-800 dark:hover:text-gray-200"><svg class="sm:size-3.5 size-3" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----></span> <span dir="auto">Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-7 max-sm:h-7 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible sm:size-3.5 size-3 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg><!----></button></div> <!--[-1--><!--]--></div><!----> <!--[0--><h1 class="relative group"><a id="paraattention" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#paraattention"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->ParaAttention<!----></span></h1><!--]--><!----> <div class="flex justify-center"><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/para-attn/flux-performance.png"/></div> <div class="flex justify-center"><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/para-attn/hunyuan-video-performance.png"/></div> <p>قد تُشكّل نماذج توليد الصور والفيديو الكبيرة، مثل <a href="https://huggingface.co/black-forest-labs/FLUX.1-dev" rel="nofollow">FLUX.1-dev</a> و<a href="https://huggingface.co/tencent/HunyuanVideo" rel="nofollow">HunyuanVideo</a>، تحدياً في الاستدلال للتطبيقات الفورية والنشر بسبب حجمها.</p> <p><a href="https://github.com/chengzeyi/ParaAttention" rel="nofollow">ParaAttention</a> هي مكتبة تنفّذ <strong>توازي السياق</strong> و<strong>تخزين كتلة البداية مؤقتًا</strong>، ويمكن دمجها مع تقنيات أخرى (torch.compile، fp8 dynamic quantization) لتسريع الاستدلال.</p> <p>ستُظهر لك هذه الدليل كيفية تطبيق ParaAttention على FLUX.1-dev و HunyuanVideo على وحدات معالجة الرسوميات NVIDIA L20.
لا تُطبَّق أي تحسينات على معيارنا المرجعي الأساسي، باستثناء HunyuanVideo لتجنب أخطاء نفاد الذاكرة.</p> <p>يُظهر معيارنا الأساسي أن FLUX.1-dev قادر على توليد صورة بدقة 1024x1024 في 28 خطوة خلال 26.36 ثانية، وأن HunyuanVideo قادر على توليد 129 إطارًا بدقة 720p في 30 خطوة خلال 3675.71 ثانية.</p> <blockquote class="tip"><p>للحصول على استنتاج أسرع حتى مع توازٍ سياقي، جرّب استخدام وحدات معالجة رسومية من NVIDIA A100 أو H100 (إن توفرت) مع دعم NVLink، خاصةً عند وجود عدد كبير من وحدات المعالجة الرسومية.</p></blockquote> <!--[1--><h2 class="relative group"><a id="first-block-cache" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#first-block-cache"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->ذاكرة التخزين المؤقت للكتلة الأولى<!----></span></h2><!--]--><!----> <p>يُقلّل تخزين مخرجات كتل transformers في النموذج وإعادة استخدامها في خطوات الاستدلال التالية من تكلفة الحساب ويجعل الاستدلال أسرع.</p> <p>ومع ذلك، يصعب تحديد متى ينبغي إعادة استخدام ذاكرة التخزين المؤقت لضمان جودة الصور أو مقاطع الفيديو المولّدة. يستخدم ParaAttention مباشرةً <strong>الفرق المتبقي في مخرج أول كتلة تحويل</strong> لتقريب الفرق بين مخرجات النموذج. عندما يكون الفرق صغيرًا بما يكفي، يُعاد استخدام الفرق المتبقي من خطوات الاستدلال السابقة. وبعبارة أخرى، يتم تجاوز خطوة إزالة الضوضاء.</p> <p>يُحقق هذا تسريعًا بمقدار مرتين في استنتاج FLUX.1-dev وHunyuanVideo مع جودة ممتازة جدًا.</p> <figure><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/para-attn/ada-cache.png" alt="Cache in Diffusion Transformer"/> <figcaption>كيف يعمل AdaCache، حيث يُعد First Block Cache أحد متغيرات</figcaption></figure> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"><!--[--><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd border-gray-800 bg-black dark:bg-gray-700 text-white">FLUX-1.dev</div><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd text-gray-500 cursor-pointer opacity-90 hover:text-gray-700 dark:hover:text-gray-200 hover:shadow-sm">HunyuanVideo</div><!--]--></div> <div class="language-select"><div><p>لتطبيق ذاكرة التخزين المؤقت للكتلة الأولى على FLUX.1-dev، استدعِ <code>apply_cache_on_pipe</code> كما هو موضح أدناه. القيمة الافتراضية لفرق البقايا في نماذج FLUX هي 0.08.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> تم النسخ</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> time
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> FluxPipeline

pipe = FluxPipeline.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    dtype=torch.bfloat16,
).to(<span class="hljs-string">&quot;cuda&quot;</span>)  <span class="hljs-comment"># or &quot;mps&quot;, &quot;xpu&quot;, &quot;cpu&quot;</span>

<span class="hljs-keyword">from</span> para_attn.first_block_cache.diffusers_adapters <span class="hljs-keyword">import</span> apply_cache_on_pipe

apply_cache_on_pipe(pipe, residual_diff_threshold=<span class="hljs-number">0.08</span>)

<span class="hljs-comment"># Enable memory savings</span>
<span class="hljs-comment"># pipe.enable_model_cpu_offload()</span>
<span class="hljs-comment"># pipe.enable_sequential_cpu_offload()</span>

begin = time.time()
image = pipe(
    <span class="hljs-string">&quot;A cat holding a sign that says hello world&quot;</span>,
    num_inference_steps=<span class="hljs-number">28</span>,
).images[<span class="hljs-number">0</span>]
end = time.time()
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Time: <span class="hljs-subst">{end - begin:<span class="hljs-number">.2</span>f}</span>s&quot;</span>)

<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Saving image to flux.png&quot;</span>)
image.save(<span class="hljs-string">&quot;flux.png&quot;</span>)<!----></pre></div><!----> <table><thead><tr><th>التحسينات</th><th>الأصل</th><th>FBCache rdt=0.06</th><th>FBCache rdt=0.08</th><th>FBCache rdt=0.10</th><th>FBCache rdt=0.12</th></tr></thead><tbody><tr><td>معاينة</td><td><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/para-attn/flux-original.png" alt="Original"/></td><td><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/para-attn/flux-fbc-0.06.png" alt="FBCache rdt=0.06"/></td><td><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/para-attn/flux-fbc-0.08.png" alt="FBCache rdt=0.08"/></td><td><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/para-attn/flux-fbc-0.10.png" alt="FBCache rdt=0.10"/></td><td><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/para-attn/flux-fbc-0.12.png" alt="FBCache rdt=0.12"/></td></tr><tr><td>الوقت الحقيقي (ثانية)</td><td>26.36</td><td>21.83</td><td>17.01</td><td>16.00</td><td>13.78</td></tr></tbody></table> <p>قلّص كاش الكتلة الأولى (First Block Cache) زمن الاستدلال إلى 17.01 ثانية مقارنةً بالخط الأساس، أي أسرع بمقدار 1.55 مرة، مع الحفاظ على فقدان جودة شبه معدوم.</p><!----></div><!----> <div hidden=""><p>لتطبيق First Block Cache على HunyuanVideo، استخدم <code>apply_cache_on_pipe</code> كما هو موضح أدناه. القيمة الافتراضية لفرق البقايا في نماذج HunyuanVideo هي 0.06.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> تم النسخ</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> time
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> HunyuanVideoPipeline, HunyuanVideoTransformer3DModel
<span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video

model_id = <span class="hljs-string">&quot;tencent/HunyuanVideo&quot;</span>
transformer = HunyuanVideoTransformer3DModel.from_pretrained(
    model_id,
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    dtype=torch.bfloat16,
    revision=<span class="hljs-string">&quot;refs/pr/18&quot;</span>,
)
pipe = HunyuanVideoPipeline.from_pretrained(
    model_id,
    transformer=transformer,
    dtype=torch.float16,
    revision=<span class="hljs-string">&quot;refs/pr/18&quot;</span>,
).to(<span class="hljs-string">&quot;cuda&quot;</span>)  <span class="hljs-comment"># or &quot;mps&quot;, &quot;xpu&quot;, &quot;cpu&quot;</span>

<span class="hljs-keyword">from</span> para_attn.first_block_cache.diffusers_adapters <span class="hljs-keyword">import</span> apply_cache_on_pipe

apply_cache_on_pipe(pipe, residual_diff_threshold=<span class="hljs-number">0.6</span>)

pipe.vae.enable_tiling()

begin = time.time()
output = pipe(
    prompt=<span class="hljs-string">&quot;A cat walks on the grass, realistic&quot;</span>,
    height=<span class="hljs-number">720</span>,
    width=<span class="hljs-number">1280</span>,
    num_frames=<span class="hljs-number">129</span>,
    num_inference_steps=<span class="hljs-number">30</span>,
).frames[<span class="hljs-number">0</span>]
end = time.time()
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Time: <span class="hljs-subst">{end - begin:<span class="hljs-number">.2</span>f}</span>s&quot;</span>)

<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Saving video to hunyuan_video.mp4&quot;</span>)
export_to_video(output, <span class="hljs-string">&quot;hunyuan_video.mp4&quot;</span>, fps=<span class="hljs-number">15</span>)<!----></pre></div><!----> <video controls=""><source src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/para-attn/hunyuan-video-original.mp4" type="video/mp4"/> Your browser does not support the video tag.</video> <small>HunyuanVideo without FBCache</small> <video controls=""><source src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/para-attn/hunyuan-video-fbc.mp4" type="video/mp4"/> Your browser does not support the video tag.</video> <small>HunyuanVideo with FBCache</small> <p>قلّص كاش الكتلة الأولى زمن الاستدلال إلى 2271.06 ثانية مقارنةً بالخط الأساسي، أي أسرع بمقدار 1.62 مرة، مع الحفاظ على فقدان جودة شبه معدوم.</p><!----></div><!----><!----></div><!----> <!--[1--><h2 class="relative group"><a id="fp8-quantization" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#fp8-quantization"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->تكميم fp8<!----></span></h2><!--]--><!----> <p>يسرّع fp8 مع التكميم الديناميكي الاستدلال بشكل أكبر ويقلل من استخدام الذاكرة. يجب تكميم كل من التفعيلات والأوزان لاستخدام نوى <a href="https://www.nvidia.com/en-us/data-center/tensor-cores/" rel="nofollow">NVIDIA Tensor Cores</a> ذات 8 بت.</p> <p>استخدم <code>float8_weight_only</code> و<code>float8_dynamic_activation_float8_weight</code> لتكميم نموذج مُشفِّر النص (text encoder) ونموذج التحويل (transformer).</p> <p>طريقة التكميم الافتراضية هي التكميم لكل مشغّل (per tensor quantization)، ولكن إذا كانت وحدة معالجة الرسوميات (GPU) لديك تدعم التكميم على مستوى الصف (row-wise quantization)، فيمكنك أيضًا تجربتها للحصول على دقة أفضل.</p> <p>قم بتثبيت <a href="https://github.com/pytorch/ao/tree/main" rel="nofollow">torchao</a> باستخدام الأمر أدناه.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> تم النسخ</div><!----></button><!----></div> <pre class="language-bash "><!---->pip3 install -U torch torchao<!----></pre></div><!----> <p>يختار <a href="https://pytorch.org/tutorials/intermediate/torch_compile_tutorial.html" rel="nofollow">torch.compile</a> مع <code>mode="max-autotune-no-cudagraphs"</code> أو <code>mode="max-autotune"</code> أفضل نواة للأداء. قد يستغرق التجميع وقتًا طويلاً إذا كانت هذه أول مرة يُستدعى فيها النموذج، لكنه يستحق العناء بعد اكتمال تجميع النموذج.</p> <p>هذا المثال يقوم بتكميم نموذج التحويل فقط، لكن يمكنك أيضًا تكميم مشفّر النص لتقليل استخدام الذاكرة بشكل أكبر.</p> <blockquote class="tip"><p>يمكن أن يُحدث التكميم الديناميكي تغييرًا كبيرًا في توزيع مخرجات النموذج، لذا عليك تغيير <code>residual_diff_threshold</code> إلى قيمة أكبر حتى يسري مفعوله.</p></blockquote> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"><!--[--><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd border-gray-800 bg-black dark:bg-gray-700 text-white">FLUX-1.dev</div><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd text-gray-500 cursor-pointer opacity-90 hover:text-gray-700 dark:hover:text-gray-200 hover:shadow-sm">HunyuanVideo</div><!--]--></div> <div class="language-select"><div><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> تم النسخ</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> time
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> FluxPipeline

pipe = FluxPipeline.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    dtype=torch.bfloat16,
).to(<span class="hljs-string">&quot;cuda&quot;</span>)  <span class="hljs-comment"># or &quot;mps&quot;, &quot;xpu&quot;, &quot;cpu&quot;</span>

<span class="hljs-keyword">from</span> para_attn.first_block_cache.diffusers_adapters <span class="hljs-keyword">import</span> apply_cache_on_pipe

apply_cache_on_pipe(
    pipe,
    residual_diff_threshold=<span class="hljs-number">0.12</span>,  <span class="hljs-comment"># Use a larger value to make the cache take effect</span>
)

<span class="hljs-keyword">from</span> torchao.quantization <span class="hljs-keyword">import</span> quantize_, float8_dynamic_activation_float8_weight, float8_weight_only

quantize_(pipe.text_encoder, float8_weight_only())
quantize_(pipe.transformer, float8_dynamic_activation_float8_weight())
pipe.transformer = torch.<span class="hljs-built_in">compile</span>(
   pipe.transformer, mode=<span class="hljs-string">&quot;max-autotune-no-cudagraphs&quot;</span>,
)

<span class="hljs-comment"># Enable memory savings</span>
<span class="hljs-comment"># pipe.enable_model_cpu_offload()</span>
<span class="hljs-comment"># pipe.enable_sequential_cpu_offload()</span>

<span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">2</span>):
    begin = time.time()
    image = pipe(
        <span class="hljs-string">&quot;A cat holding a sign that says hello world&quot;</span>,
        num_inference_steps=<span class="hljs-number">28</span>,
    ).images[<span class="hljs-number">0</span>]
    end = time.time()
    <span class="hljs-keyword">if</span> i == <span class="hljs-number">0</span>:
        <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Warm up time: <span class="hljs-subst">{end - begin:<span class="hljs-number">.2</span>f}</span>s&quot;</span>)
    <span class="hljs-keyword">else</span>:
        <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Time: <span class="hljs-subst">{end - begin:<span class="hljs-number">.2</span>f}</span>s&quot;</span>)

<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Saving image to flux.png&quot;</span>)
image.save(<span class="hljs-string">&quot;flux.png&quot;</span>)<!----></pre></div><!----> <p>قلّص كلٌّ من fp8 dynamic quantization وtorch.compile زمن الاستدلال إلى 7.56 ثانية مقارنةً بالخط الأساس، أي أسرع بمقدار 3.48 مرة.</p><!----></div><!----> <div hidden=""><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> تم النسخ</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> time
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> HunyuanVideoPipeline, HunyuanVideoTransformer3DModel
<span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video

model_id = <span class="hljs-string">&quot;tencent/HunyuanVideo&quot;</span>
transformer = HunyuanVideoTransformer3DModel.from_pretrained(
    model_id,
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    dtype=torch.bfloat16,
    revision=<span class="hljs-string">&quot;refs/pr/18&quot;</span>,
)
pipe = HunyuanVideoPipeline.from_pretrained(
    model_id,
    transformer=transformer,
    dtype=torch.float16,
    revision=<span class="hljs-string">&quot;refs/pr/18&quot;</span>,
).to(<span class="hljs-string">&quot;cuda&quot;</span>)  <span class="hljs-comment"># or &quot;mps&quot;, &quot;xpu&quot;, &quot;cpu&quot;</span>

<span class="hljs-keyword">from</span> para_attn.first_block_cache.diffusers_adapters <span class="hljs-keyword">import</span> apply_cache_on_pipe

apply_cache_on_pipe(pipe)

<span class="hljs-keyword">from</span> torchao.quantization <span class="hljs-keyword">import</span> quantize_, float8_dynamic_activation_float8_weight, float8_weight_only

quantize_(pipe.text_encoder, float8_weight_only())
quantize_(pipe.transformer, float8_dynamic_activation_float8_weight())
pipe.transformer = torch.<span class="hljs-built_in">compile</span>(
   pipe.transformer, mode=<span class="hljs-string">&quot;max-autotune-no-cudagraphs&quot;</span>,
)

<span class="hljs-comment"># Enable memory savings</span>
pipe.vae.enable_tiling()
<span class="hljs-comment"># pipe.enable_model_cpu_offload()</span>
<span class="hljs-comment"># pipe.enable_sequential_cpu_offload()</span>

<span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">2</span>):
    begin = time.time()
    output = pipe(
        prompt=<span class="hljs-string">&quot;A cat walks on the grass, realistic&quot;</span>,
        height=<span class="hljs-number">720</span>,
        width=<span class="hljs-number">1280</span>,
        num_frames=<span class="hljs-number">129</span>,
        num_inference_steps=<span class="hljs-number">1</span> <span class="hljs-keyword">if</span> i == <span class="hljs-number">0</span> <span class="hljs-keyword">else</span> <span class="hljs-number">30</span>,
    ).frames[<span class="hljs-number">0</span>]
    end = time.time()
    <span class="hljs-keyword">if</span> i == <span class="hljs-number">0</span>:
        <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Warm up time: <span class="hljs-subst">{end - begin:<span class="hljs-number">.2</span>f}</span>s&quot;</span>)
    <span class="hljs-keyword">else</span>:
        <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Time: <span class="hljs-subst">{end - begin:<span class="hljs-number">.2</span>f}</span>s&quot;</span>)

<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Saving video to hunyuan_video.mp4&quot;</span>)
export_to_video(output, <span class="hljs-string">&quot;hunyuan_video.mp4&quot;</span>, fps=<span class="hljs-number">15</span>)<!----></pre></div><!----> <p>تتميز بطاقة NVIDIA L20 بذاكرة تبلغ 48 جيجابايت فقط، وقد تواجه أخطاء نفاد الذاكرة (OOM) بعد التجميع إذا لم يتم استدعاء <code>enable_model_cpu_offload</code>، وذلك لأن HunyuanVideo يحتوي على متجهات تنشيط كبيرة جداً عند التشغيل بدقة عالية وعدد كبير من الإطارات. بالنسبة للبطاقات التي تقل ذاكرتها عن 80 جيجابايت، يمكنك محاولة تقليل الدقة وعدد الإطارات لتجنب أخطاء OOM.</p> <p>عادةً ما تكون نماذج توليد الفيديو الكبيرة مقيدة بحسابات الانتباه أكثر من الطبقات المتصلة بالكامل. ولا تستفيد هذه النماذج بشكل كبير من التكميم وtorch.compile.</p><!----></div><!----><!----></div><!----> <!--[1--><h2 class="relative group"><a id="context-parallelism" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#context-parallelism"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->توازي السياق<!----></span></h2><!--]--><!----> <p>يعمل توازي السياق على موازاة الاستدلال ويتوسع مع استخدام عدة وحدات معالجة رسومية. ويتيح لك التصميم التركيبي لـ ParaAttention دمج توازي السياق مع ذاكرة التخزين المؤقت للكتلة الأولى والكمّية الديناميكية.</p> <blockquote class="tip"><p>راجع مستودع <a href="https://github.com/chengzeyi/ParaAttention/tree/main" rel="nofollow">ParaAttention</a> للحصول على تعليمات مفصّلة وأمثلة حول كيفية توسيع نطاق الاستدلال باستخدام وحدات معالجة رسومية متعددة.</p></blockquote> <p>إذا كان من الضروري أن تكون عملية الاستدلال مستمرة وقابلة للخدمة، يُقترح استخدام <a href="https://pytorch.org/docs/stable/multiprocessing.html" rel="nofollow">torch.multiprocessing</a> لكتابة معالج استدلال خاص بك. يمكن أن يؤدي ذلك إلى إزالة التكلفة الإضافية المرتبطة بإطلاق العملية وتحميل النموذج وإعادة تجميعه.</p> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"><!--[--><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd border-gray-800 bg-black dark:bg-gray-700 text-white">FLUX-1.dev</div><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd text-gray-500 cursor-pointer opacity-90 hover:text-gray-700 dark:hover:text-gray-200 hover:shadow-sm">HunyuanVideo</div><!--]--></div> <div class="language-select"><div><p>يجمع المثال البرمجي أدناه بين First Block Cache، والتكميم الديناميكي fp8، وtorch.compile، وتوازي السياق (Context Parallelism) للحصول على أسرع سرعة استدلال.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> تم النسخ</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> time
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">import</span> torch.distributed <span class="hljs-keyword">as</span> dist
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> FluxPipeline

dist.init_process_group()

torch.cuda.set_device(dist.get_rank())

pipe = FluxPipeline.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    dtype=torch.bfloat16,
).to(<span class="hljs-string">&quot;cuda&quot;</span>)  <span class="hljs-comment"># or &quot;mps&quot;, &quot;xpu&quot;, &quot;cpu&quot;</span>

<span class="hljs-keyword">from</span> para_attn.context_parallel <span class="hljs-keyword">import</span> init_context_parallel_mesh
<span class="hljs-keyword">from</span> para_attn.context_parallel.diffusers_adapters <span class="hljs-keyword">import</span> parallelize_pipe
<span class="hljs-keyword">from</span> para_attn.parallel_vae.diffusers_adapters <span class="hljs-keyword">import</span> parallelize_vae

mesh = init_context_parallel_mesh(
    pipe.device.<span class="hljs-built_in">type</span>,
    max_ring_dim_size=<span class="hljs-number">2</span>,
)
parallelize_pipe(
    pipe,
    mesh=mesh,
)
parallelize_vae(pipe.vae, mesh=mesh._flatten())

<span class="hljs-keyword">from</span> para_attn.first_block_cache.diffusers_adapters <span class="hljs-keyword">import</span> apply_cache_on_pipe

apply_cache_on_pipe(
    pipe,
    residual_diff_threshold=<span class="hljs-number">0.12</span>,  <span class="hljs-comment"># Use a larger value to make the cache take effect</span>
)

<span class="hljs-keyword">from</span> torchao.quantization <span class="hljs-keyword">import</span> quantize_, float8_dynamic_activation_float8_weight, float8_weight_only

quantize_(pipe.text_encoder, float8_weight_only())
quantize_(pipe.transformer, float8_dynamic_activation_float8_weight())
torch._inductor.config.reorder_for_compute_comm_overlap = <span class="hljs-literal">True</span>
pipe.transformer = torch.<span class="hljs-built_in">compile</span>(
   pipe.transformer, mode=<span class="hljs-string">&quot;max-autotune-no-cudagraphs&quot;</span>,
)

<span class="hljs-comment"># Enable memory savings</span>
<span class="hljs-comment"># pipe.enable_model_cpu_offload(gpu_id=dist.get_rank())</span>
<span class="hljs-comment"># pipe.enable_sequential_cpu_offload(gpu_id=dist.get_rank())</span>

<span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">2</span>):
    begin = time.time()
    image = pipe(
        <span class="hljs-string">&quot;A cat holding a sign that says hello world&quot;</span>,
        num_inference_steps=<span class="hljs-number">28</span>,
        output_type=<span class="hljs-string">&quot;pil&quot;</span> <span class="hljs-keyword">if</span> dist.get_rank() == <span class="hljs-number">0</span> <span class="hljs-keyword">else</span> <span class="hljs-string">&quot;pt&quot;</span>,
    ).images[<span class="hljs-number">0</span>]
    end = time.time()
    <span class="hljs-keyword">if</span> dist.get_rank() == <span class="hljs-number">0</span>:
        <span class="hljs-keyword">if</span> i == <span class="hljs-number">0</span>:
            <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Warm up time: <span class="hljs-subst">{end - begin:<span class="hljs-number">.2</span>f}</span>s&quot;</span>)
        <span class="hljs-keyword">else</span>:
            <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Time: <span class="hljs-subst">{end - begin:<span class="hljs-number">.2</span>f}</span>s&quot;</span>)

<span class="hljs-keyword">if</span> dist.get_rank() == <span class="hljs-number">0</span>:
    <span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Saving image to flux.png&quot;</span>)
    image.save(<span class="hljs-string">&quot;flux.png&quot;</span>)

dist.destroy_process_group()<!----></pre></div><!----> <p>احفظه في <code>run_flux.py</code> ثم شغّله باستخدام <a href="https://pytorch.org/docs/stable/elastic/run.html" rel="nofollow">torchrun</a>.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> تم النسخ</div><!----></button><!----></div> <pre class="language-bash "><!----><span class="hljs-comment"># Use --nproc_per_node to specify the number of GPUs</span>
torchrun --nproc_per_node=2 run_flux.py<!----></pre></div><!----> <p>تُقلَّص سرعة الاستدلال إلى 8.20 ثانية مقارنةً بالخط الأساسي، أي أسرع بـ 3.21 مرة، باستخدام معالجي رسومات NVIDIA L20. أما على 4 معالجات L20، فتبلغ سرعة الاستدلال 3.90 ثانية، أي أسرع بـ 6.75 مرة.</p><!----></div><!----> <div hidden=""><p>يجمع المثال البرمجي أدناه بين First Block Cache وContext Parallelism للحصول على أسرع سرعة استدلال.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> تم النسخ</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> time
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">import</span> torch.distributed <span class="hljs-keyword">as</span> dist
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> HunyuanVideoPipeline, HunyuanVideoTransformer3DModel
<span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video

dist.init_process_group()

torch.cuda.set_device(dist.get_rank())

model_id = <span class="hljs-string">&quot;tencent/HunyuanVideo&quot;</span>
transformer = HunyuanVideoTransformer3DModel.from_pretrained(
    model_id,
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    dtype=torch.bfloat16,
    revision=<span class="hljs-string">&quot;refs/pr/18&quot;</span>,
)
pipe = HunyuanVideoPipeline.from_pretrained(
    model_id,
    transformer=transformer,
    dtype=torch.float16,
    revision=<span class="hljs-string">&quot;refs/pr/18&quot;</span>,
).to(<span class="hljs-string">&quot;cuda&quot;</span>)  <span class="hljs-comment"># or &quot;mps&quot;, &quot;xpu&quot;, &quot;cpu&quot;</span>

<span class="hljs-keyword">from</span> para_attn.context_parallel <span class="hljs-keyword">import</span> init_context_parallel_mesh
<span class="hljs-keyword">from</span> para_attn.context_parallel.diffusers_adapters <span class="hljs-keyword">import</span> parallelize_pipe
<span class="hljs-keyword">from</span> para_attn.parallel_vae.diffusers_adapters <span class="hljs-keyword">import</span> parallelize_vae

mesh = init_context_parallel_mesh(
    pipe.device.<span class="hljs-built_in">type</span>,
)
parallelize_pipe(
    pipe,
    mesh=mesh,
)
parallelize_vae(pipe.vae, mesh=mesh._flatten())

<span class="hljs-keyword">from</span> para_attn.first_block_cache.diffusers_adapters <span class="hljs-keyword">import</span> apply_cache_on_pipe

apply_cache_on_pipe(pipe)

<span class="hljs-comment"># from torchao.quantization import quantize_, float8_dynamic_activation_float8_weight, float8_weight_only</span>
<span class="hljs-comment">#</span>
<span class="hljs-comment"># torch._inductor.config.reorder_for_compute_comm_overlap = True</span>
<span class="hljs-comment">#</span>
<span class="hljs-comment"># quantize_(pipe.text_encoder, float8_weight_only())</span>
<span class="hljs-comment"># quantize_(pipe.transformer, float8_dynamic_activation_float8_weight())</span>
<span class="hljs-comment"># pipe.transformer = torch.compile(</span>
<span class="hljs-comment">#    pipe.transformer, mode=&quot;max-autotune-no-cudagraphs&quot;,</span>
<span class="hljs-comment"># )</span>

<span class="hljs-comment"># Enable memory savings</span>
pipe.vae.enable_tiling()
<span class="hljs-comment"># pipe.enable_model_cpu_offload(gpu_id=dist.get_rank())</span>
<span class="hljs-comment"># pipe.enable_sequential_cpu_offload(gpu_id=dist.get_rank())</span>

<span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">2</span>):
    begin = time.time()
    output = pipe(
        prompt=<span class="hljs-string">&quot;A cat walks on the grass, realistic&quot;</span>,
        height=<span class="hljs-number">720</span>,
        width=<span class="hljs-number">1280</span>,
        num_frames=<span class="hljs-number">129</span>,
        num_inference_steps=<span class="hljs-number">1</span> <span class="hljs-keyword">if</span> i == <span class="hljs-number">0</span> <span class="hljs-keyword">else</span> <span class="hljs-number">30</span>,
        output_type=<span class="hljs-string">&quot;pil&quot;</span> <span class="hljs-keyword">if</span> dist.get_rank() == <span class="hljs-number">0</span> <span class="hljs-keyword">else</span> <span class="hljs-string">&quot;pt&quot;</span>,
    ).frames[<span class="hljs-number">0</span>]
    end = time.time()
    <span class="hljs-keyword">if</span> dist.get_rank() == <span class="hljs-number">0</span>:
        <span class="hljs-keyword">if</span> i == <span class="hljs-number">0</span>:
            <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Warm up time: <span class="hljs-subst">{end - begin:<span class="hljs-number">.2</span>f}</span>s&quot;</span>)
        <span class="hljs-keyword">else</span>:
            <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Time: <span class="hljs-subst">{end - begin:<span class="hljs-number">.2</span>f}</span>s&quot;</span>)

<span class="hljs-keyword">if</span> dist.get_rank() == <span class="hljs-number">0</span>:
    <span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Saving video to hunyuan_video.mp4&quot;</span>)
    export_to_video(output, <span class="hljs-string">&quot;hunyuan_video.mp4&quot;</span>, fps=<span class="hljs-number">15</span>)

dist.destroy_process_group()<!----></pre></div><!----> <p>احفظه في <code>run_hunyuan_video.py</code> ثم شغّله باستخدام <a href="https://pytorch.org/docs/stable/elastic/run.html" rel="nofollow">torchrun</a>.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> تم النسخ</div><!----></button><!----></div> <pre class="language-bash "><!----><span class="hljs-comment"># Use --nproc_per_node to specify the number of GPUs</span>
torchrun --nproc_per_node=8 run_hunyuan_video.py<!----></pre></div><!----> <p>تُقلَّص سرعة الاستدلال إلى 649.23 ثانية مقارنةً بالخط الأساسي، أي أسرع بـ 5.66 مرة، باستخدام 8 وحدات معالجة رسومية NVIDIA L20.</p><!----></div><!----><!----></div><!----> <!--[1--><h2 class="relative group"><a id="benchmarks" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#benchmarks"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->المعايير<!----></span></h2><!--]--><!----> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"><!--[--><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd border-gray-800 bg-black dark:bg-gray-700 text-white">FLUX-1.dev</div><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd text-gray-500 cursor-pointer opacity-90 hover:text-gray-700 dark:hover:text-gray-200 hover:shadow-sm">HunyuanVideo</div><!--]--></div> <div class="language-select"><div><table><thead><tr><th>نوع وحدة معالجة الرسوميات</th><th>عدد وحدات معالجة الرسوميات</th><th>التحسينات</th><th>الوقت الحقيقي (ثانية)</th><th>التسريع</th></tr></thead><tbody><tr><td>NVIDIA L20</td><td>1</td><td>الخط الأساسي</td><td>26.36</td><td>1.00x</td></tr><tr><td>NVIDIA L20</td><td>1</td><td>FBCache (rdt=0.08)</td><td>17.01</td><td>1.55x</td></tr><tr><td>NVIDIA L20</td><td>1</td><td>FP8 DQ</td><td>13.40</td><td>1.96x</td></tr><tr><td>NVIDIA L20</td><td>1</td><td>FBCache (rdt=0.12) + FP8 DQ</td><td>7.56</td><td>3.48x</td></tr><tr><td>NVIDIA L20</td><td>2</td><td>FBCache (rdt=0.12) + FP8 DQ + CP</td><td>4.92</td><td>5.35x</td></tr><tr><td>NVIDIA L20</td><td>4</td><td>FBCache (rdt=0.12) + FP8 DQ + CP</td><td>3.90</td><td>6.75x</td></tr></tbody></table><!----></div><!----> <div hidden=""><table><thead><tr><th>نوع وحدة معالجة الرسوميات</th><th>عدد وحدات معالجة الرسوميات</th><th>التحسينات</th><th>الوقت الحقيقي (ثانية)</th><th>التسريع</th></tr></thead><tbody><tr><td>NVIDIA L20</td><td>1</td><td>الخط الأساسي</td><td>3675.71</td><td>1.00x</td></tr><tr><td>NVIDIA L20</td><td>1</td><td>FBCache</td><td>2271.06</td><td>1.62x</td></tr><tr><td>NVIDIA L20</td><td>2</td><td>FBCache + CP</td><td>1132.90</td><td>3.24x</td></tr><tr><td>NVIDIA L20</td><td>4</td><td>FBCache + CP</td><td>718.15</td><td>5.12x</td></tr><tr><td>NVIDIA L20</td><td>8</td><td>FBCache + CP</td><td>649.23</td><td>5.66x</td></tr></tbody></table><!----></div><!----><!----></div><!----> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/diffusers/blob/main/docs/source/en/optimization/para_attn.md" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg><!----> <span><span class="underline">Update</span> on GitHub</span></a><!----> <p></p><!--]--><!----><!--]--><!--]--><!--]--> <!--[-1--><!--]--><!--]-->
			
			<script>
				{
					__sveltekit_1qsdbwf = {
						base: "/docs/diffusers/main/en",
						assets: "/docs/diffusers/main/en"
					};
					const element = document.currentScript.parentElement;
					Promise.all([
						import("/docs/diffusers/main/en/_app/immutable/entry/start.0S0_iEoG.js"),
						import("/docs/diffusers/main/en/_app/immutable/entry/app.CPdOxCbw.js")
					]).then(([kit, app]) => {
						kit.start(app, element, {
							node_ids: [0, 344],
							data: [null,null],
							form: null,
							error: null
						});
					});
				}
			</script>
		
Xet Storage Details

Size:: 57.6 kB
Xet hash:: 17171b9f51cf203930ea8c394ecdc4df2273b506fe6d0401c3d0b84ac2f8a276
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.