Buckets:
| <meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"Cosmos 3","local":"cosmos-3","sections":[{"title":"Apa yang baru di Cosmos 3","local":"whats-new-in-cosmos-3","sections":[],"depth":2},{"title":"Checkpoint yang tersedia","local":"available-checkpoints","sections":[],"depth":2},{"title":"Denoising campuran FP8 W8A8/W8A16","local":"fp8-mixed-w8a8w8a16-denoising","sections":[],"depth":2},{"title":"Upsampling prompt","local":"prompt-upsampling","sections":[],"depth":2},{"title":"Teks-ke-video","local":"text-to-video","sections":[],"depth":2},{"title":"Teks-ke-gambar","local":"text-to-image","sections":[],"depth":2},{"title":"Gambar-ke-video","local":"image-to-video","sections":[],"depth":2},{"title":"Video-ke-video","local":"video-to-video","sections":[],"depth":2},{"title":"Video-ke-video dengan suara","local":"video-to-video-with-sound","sections":[],"depth":2},{"title":"Teks-ke-video dengan suara","local":"text-to-video-with-sound","sections":[],"depth":2},{"title":"Generasi yang dikondisikan oleh aksi","local":"action-conditioned-generation","sections":[{"title":"Kebijakan tindakan","local":"action-policy","sections":[],"depth":3}],"depth":2},{"title":"SeaCache","local":"seacache","sections":[],"depth":2},{"title":"Paralelisme konteks","local":"context-parallelism","sections":[{"title":"Jalankan","local":"run-it","sections":[],"depth":3},{"title":"Melatih model besar dengan paralelisme tensor","local":"fitting-large-models-with-tensor-parallelism","sections":[],"depth":3},{"title":"Gunakan di pipeline modular Anda sendiri","local":"use-it-in-your-own-modular-pipeline","sections":[],"depth":3}],"depth":2},{"title":"Template metadata","local":"metadata-templates","sections":[],"depth":2},{"title":"Pemeriksa keamanan","local":"safety-checker","sections":[],"depth":2},{"title":"Cosmos3OmniPipeline","local":"diffusers.Cosmos3OmniPipeline","sections":[],"depth":2},{"title":"Cosmos3OmniModularPipeline","local":"cosmos3omnimodularpipeline","sections":[{"title":"Contoh modular untuk semua workflow yang ada","local":"modular-examples-for-all-existing-workflows","sections":[],"depth":3},{"title":"Transfer modular (kontrol struktural)","local":"modular-transfer-structural-control","sections":[],"depth":3},{"title":"Teks-ke-gambar dan gambar-ke-video yang telah didistilasi (sedikit langkah)","local":"diffusers.Cosmos3OmniModularPipeline","sections":[],"depth":3}],"depth":2},{"title":"Cosmos3DistilledModularPipeline","local":"diffusers.Cosmos3DistilledModularPipeline","sections":[],"depth":2},{"title":"CosmosActionCondition","local":"diffusers.CosmosActionCondition","sections":[],"depth":2},{"title":"Cosmos3OmniPipelineOutput","local":"diffusers.pipelines.cosmos.pipeline_cosmos3_omni.Cosmos3OmniPipelineOutput","sections":[],"depth":2}],"depth":1}"/> | |
| <link href="/docs/diffusers/main/en/_app/immutable/entry/start.U6TUchwJ.js" rel="modulepreload"> | |
| <link href="/docs/diffusers/main/en/_app/immutable/chunks/yxSHiGDF.js" rel="modulepreload"> | |
| <link href="/docs/diffusers/main/en/_app/immutable/chunks/CPfOwUO_.js" rel="modulepreload"> | |
| <link href="/docs/diffusers/main/en/_app/immutable/entry/app.3qKhPoX-.js" rel="modulepreload"> | |
| <link href="/docs/diffusers/main/en/_app/immutable/chunks/BYe-f74j.js" rel="modulepreload"> | |
| <link href="/docs/diffusers/main/en/_app/immutable/chunks/DnVIJfC5.js" rel="modulepreload"> | |
| <link href="/docs/diffusers/main/en/_app/immutable/chunks/DsnmJJEf.js" rel="modulepreload"> | |
| <link href="/docs/diffusers/main/en/_app/immutable/chunks/D32hXVzh.js" rel="modulepreload"> | |
| <link href="/docs/diffusers/main/en/_app/immutable/nodes/0.DqpuBF3n.js" rel="modulepreload"> | |
| <link href="/docs/diffusers/main/en/_app/immutable/chunks/Dyp9xKq8.js" rel="modulepreload"> | |
| <link href="/docs/diffusers/main/en/_app/immutable/nodes/7.MVmUqGUj.js" rel="modulepreload"> | |
| <link href="/docs/diffusers/main/en/_app/immutable/chunks/Dk44y1gU.js" rel="modulepreload"> | |
| <link href="/docs/diffusers/main/en/_app/immutable/chunks/CYniLWKs.js" rel="modulepreload"> | |
| <!--if5dit--><meta name="hf:doc:metadata" content="{"title":"Cosmos 3","local":"cosmos-3","sections":[{"title":"What’s new in Cosmos 3","local":"whats-new-in-cosmos-3","sections":[],"depth":2},{"title":"Available checkpoints","local":"available-checkpoints","sections":[],"depth":2},{"title":"FP8 mixed W8A8/W8A16 denoising","local":"fp8-mixed-w8a8w8a16-denoising","sections":[],"depth":2},{"title":"Prompt upsampling","local":"prompt-upsampling","sections":[],"depth":2},{"title":"Text-to-video","local":"text-to-video","sections":[],"depth":2},{"title":"Text-to-image","local":"text-to-image","sections":[],"depth":2},{"title":"Image-to-video","local":"image-to-video","sections":[],"depth":2},{"title":"Video-to-video","local":"video-to-video","sections":[],"depth":2},{"title":"Video-to-video with sound","local":"video-to-video-with-sound","sections":[],"depth":2},{"title":"Text-to-video with sound","local":"text-to-video-with-sound","sections":[],"depth":2},{"title":"Action-conditioned generation","local":"action-conditioned-generation","sections":[{"title":"Action policy","local":"action-policy","sections":[],"depth":3}],"depth":2},{"title":"SeaCache","local":"seacache","sections":[],"depth":2},{"title":"Context parallelism","local":"context-parallelism","sections":[{"title":"Run it","local":"run-it","sections":[],"depth":3},{"title":"Fitting large models with tensor parallelism","local":"fitting-large-models-with-tensor-parallelism","sections":[],"depth":3},{"title":"Use it in your own modular pipeline","local":"use-it-in-your-own-modular-pipeline","sections":[],"depth":3}],"depth":2},{"title":"Metadata templates","local":"metadata-templates","sections":[],"depth":2},{"title":"Safety checker","local":"safety-checker","sections":[],"depth":2},{"title":"Cosmos3OmniPipeline","local":"diffusers.Cosmos3OmniPipeline","sections":[],"depth":2},{"title":"Cosmos3OmniModularPipeline","local":"cosmos3omnimodularpipeline","sections":[{"title":"Modular examples for all existing workflows","local":"modular-examples-for-all-existing-workflows","sections":[],"depth":3},{"title":"Modular transfer (structural control)","local":"modular-transfer-structural-control","sections":[],"depth":3},{"title":"Distilled (few-step) text-to-image and image-to-video","local":"diffusers.Cosmos3OmniModularPipeline","sections":[],"depth":3}],"depth":2},{"title":"Cosmos3DistilledModularPipeline","local":"diffusers.Cosmos3DistilledModularPipeline","sections":[],"depth":2},{"title":"CosmosActionCondition","local":"diffusers.CosmosActionCondition","sections":[],"depth":2},{"title":"Cosmos3OmniPipelineOutput","local":"diffusers.pipelines.cosmos.pipeline_cosmos3_omni.Cosmos3OmniPipelineOutput","sections":[],"depth":2}],"depth":1}"/><!----> | |
| <link href="/docs/diffusers/main/en/_app/immutable/assets/0.tn0RQdqM.css" rel="modulepreload"> <!--[--><!--[0--><!--[--><!--[0--><!--[--><p></p> <!--[0--><h1 class="relative group"><a id="cosmos-3" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#cosmos-3"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Cosmos 3<!----></span></h1><!--]--><!----> <p>NVIDIA Cosmos 3 adalah model dasar dunia (WFM) terpadu untuk Physical AI — sebuah model omni tunggal yang menggabungkan generasi dunia, penalaran fisik, dan generasi aksi. Model ini menggantikan model Predict, Reason, dan Transfer yang terpisah dari rilis Cosmos sebelumnya: baik Anda membangun untuk robotika, kendaraan otonom, maupun ruang cerdas, Cosmos 3 memberikan satu fondasi untuk mensimulasikan dan memahami dunia fisik.</p> <p>Yang disertakan dalam rilis ini:</p> <ul><li>Model di Hugging Face Hub dengan kartu model dan lisensi</li> <li>Integrasi Cosmos 3 Diffusers untuk pipeline generasi (halaman ini)</li> <li>Skrip pelatihan pasca untuk penyesuaian halus Cosmos 3 pada data Anda sendiri</li> <li>Dataset generasi data sintetis (SDG) terbuka untuk AI Fisik</li></ul> <!--[1--><h2 class="relative group"><a id="whats-new-in-cosmos-3" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#whats-new-in-cosmos-3"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Apa yang baru di Cosmos 3<!----></span></h2><!--]--><!----> <p>Perubahan terbesar dari rilis Cosmos sebelumnya adalah bahwa Cosmos 3 merupakan <em>omni-model</em>, yang dibangun di atas arsitektur Mixture-of-Transformers (MoT). Sebelumnya, pengembang bekerja dengan model-model terpisah untuk world generation (Predict), controlled generation (Transfer), scene understanding (Reason), dan action-policy generation. Cosmos 3 menyatukan semua fungsi tersebut dalam satu model yang melakukan penalaran dan generasi lintas modalitas dalam satu forward pass.</p> <p>Dari satu model, Anda dapat:</p> <ul><li>Hasilkan dunia video yang secara fisik masuk akal dari teks, gambar, atau input aksi (image-to-video, text-to-video, action-conditioned video generation).</li> <li>Berpikir tentang sifat-sifat fisik seperti gerak, kausalitas, dan hubungan spasial.</li> <li>Prediksikan urutan video dan aksi masa depan dari keadaan saat ini.</li> <li>Transfer adegan antar sudut pandang dan kondisi dengan kontrol struktural <em>(segera hadir)</em>.</li></ul> <p>Di balik layar, satu <code>Cosmos3OmniTransformer</code> menjalankan model bahasa bergaya Qwen secara paralel dengan jalur generasi difusi: token teks mengalir melalui stream “pemahaman” kausal, sementara latens video dan suara mengalir melalui stream “generasi” yang diperhatikan secara bi-arah, yang disatukan oleh 3D multimodal RoPE. Lihat <a href="https://huggingface.co/papers/2501.03575" rel="nofollow">makalah Cosmos World Foundation Model Platform</a> untuk latar belakang arsitekturnya.</p> <!--[1--><h2 class="relative group"><a id="available-checkpoints" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#available-checkpoints"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Checkpoint yang tersedia<!----></span></h2><!--]--><!----> <p>Dua checkpoint dirilis di Hub — <a href="https://huggingface.co/nvidia/Cosmos3-Nano" rel="nofollow"><code>nvidia/Cosmos3-Nano</code></a> (lebih kecil, lebih cepat) dan <a href="https://huggingface.co/nvidia/Cosmos3-Super" rel="nofollow"><code>nvidia/Cosmos3-Super</code></a> (lebih besar, kualitas lebih tinggi). Kelas pipeline yang sama mendukung text-to-image, text-to-video, image-to-video, dan (dengan checkpoint yang mampu menghasilkan suara) text+image-to-video-with-sound — pilih repo dan gunakan tab per-model di masing-masing workflow di bawah.</p> <blockquote class="tip"><p>Pastikan untuk melihat <a href="../../using-diffusers/schedulers">panduan</a> Schedulers untuk mempelajari cara mengeksplorasi tradeoff antara kecepatan dan kualitas scheduler, serta lihat bagian <a href="../../using-diffusers/loading#reusing-models-in-multiple-pipelines">reuse components across pipelines</a> untuk mempelajari cara memuat komponen yang sama secara efisien ke dalam beberapa pipeline.</p></blockquote> <!--[1--><h2 class="relative group"><a id="fp8-mixed-w8a8w8a16-denoising" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#fp8-mixed-w8a8w8a16-denoising"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Denoising campuran FP8 W8A8/W8A16<!----></span></h2><!--]--><!----> <p>Checkpoint ModelOpt FP8 resmi untuk Nano, Super, dan Edge dipublikasikan sebagai repositori Hub mereka sendiri: <a href="https://huggingface.co/nvidia/Cosmos3-Nano-FP8" rel="nofollow"><code>nvidia/Cosmos3-Nano-FP8</code></a>, <a href="https://huggingface.co/nvidia/Cosmos3-Super-FP8" rel="nofollow"><code>nvidia/Cosmos3-Super-FP8</code></a>, dan <a href="https://huggingface.co/nvidia/Cosmos3-Edge-FP8" rel="nofollow"><code>nvidia/Cosmos3-Edge-FP8</code></a>. Muat <code>main</code> dari repositori FP8. <a href="https://huggingface.co/nvidia/Cosmos3-Super-Image2Video" rel="nofollow"><code>nvidia/Cosmos3-Super-Image2Video</code></a> tidak memiliki repositori FP8 terpisah; muat dengan <code>revision="fp8"</code>.</p> <p>Semua checkpoint ini dikuantisasi dengan cara yang sama. <strong>W8A8</strong> menggunakan bobot 8-bit dan aktivasi 8-bit (GEMM ModelOpt yang dipulihkan). <strong>W8A16</strong> menggunakan kembali bobot 8-bit yang sama tetapi melewati kuantisasi aktivasi: bobot FP8 didekuantisasi dan linear standar dijalankan pada aktivasi BF16/FP16/FP32.</p> <p>Menjalankan W8A8 pada setiap langkah dapat menghasilkan flickering yang terlihat pada pembuatan video multi-langkah. Oleh karena itu, checkpoint video Nano, Super, Super-I2V, dan Edge FP8 menyatakan jadwal di <code>transformer/config.json</code>: <strong>W8A16 pada 3 langkah pertama dan 3 langkah terakhir</strong>, <strong>W8A8 di tengah</strong>. Diffusers membaca jumlah tersebut dari checkpoint, bukan menghardcode-nya. Presisi dipilih sekali per langkah scheduler sehingga panggilan cond/uncond untuk classifier-free guidance cocok.</p> <p>Generasi gambar dan checkpoint terdistil beberapa langkah tidak menunjukkan flickering tersebut, sehingga Super-T2I dan repositori terdistil 4-langkah FP8 tidak mendeklarasikan jadwal apa pun dan tetap W8A8 di setiap langkah. Jadwal ini juga <strong>hanya untuk ModelOpt FP8</strong>: backend kuantisasi lain (misalnya TorchAO) mempertahankan forward native mereka.</p> <p>Muat repo FP8 dengan jalur restore yang sama seperti <a href="../../quantization/modelopt">panduan ModelOpt</a>. Repo tersebut sudah membawa konfigurasi kuantisasi. Presisi campuran kemudian mengikuti checkpoint secara otomatis:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Disalin</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniPipeline | |
| pipe = Cosmos3OmniPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Nano-FP8"</span>, | |
| dtype=torch.bfloat16, | |
| device_map=<span class="hljs-string">"cuda"</span>, | |
| ) | |
| result = pipe(prompt=<span class="hljs-string">"..."</span>, num_inference_steps=<span class="hljs-number">35</span>)<!----></pre></div><!----> <p>Dua pilihan saat generate:</p> <ul><li><strong>Default</strong> (<code>mixed_precision_format=None</code>): jika checkpoint menyatakan <code>diffusion_step_policy</code>, jalankan W8A16 pada N langkah pertama/terakhir dan W8A8 native di tengah. Itulah resep yang dimaksud untuk <strong>video</strong> FP8 multi-langkah (lebih sedikit flickering daripada all-W8A8). Distilled 4-step dan Super-T2I FP8 mengabaikan policy, sehingga default-nya sudah all W8A8.</li> <li><strong><code>mixed_precision_format="none"</code></strong>: pertahankan setiap langkah pada W8A8 native. Lebih cepat, karena W8A16 adalah dequant + <code>torch.nn.functional.linear</code> alih-alih FP8 GEMM yang dipulihkan, tetapi video multi-langkah dapat berkedip. Gunakan ini untuk A/B pada jadwal atau untuk mencocokkan baseline yang sepenuhnya terkuantisasi.</li></ul> <p>Pada satu workstation Blackwell, Cosmos3-Nano-FP8 pada 720×1280 / 35 langkah sekitar <strong>27% lebih lambat</strong> (T2I) dan <strong>13% lebih lambat</strong> (T2V 49-frame) dengan jadwal campuran bawaan dibandingkan dengan <code>"none"</code>. Angka-angka tersebut bukan jaminan throughput. Gunakan <code>"fp8"</code> hanya untuk memaksa jadwal first/last-N pada checkpoint ModelOpt FP8 yang tidak memiliki kebijakan.</p> <!--[1--><h2 class="relative group"><a id="prompt-upsampling" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#prompt-upsampling"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Upsampling prompt<!----></span></h2><!--]--><!----> <p>Cosmos 3 dilatih pada caption yang panjang dan sangat deskriptif. Untuk kualitas optimal, prompt teks pendek harus <strong>di-upsample menjadi struktur JSON tertentu</strong> sebelum diteruskan ke pipeline. Upsampler ini terdapat dalam paket <a href="https://github.com/NVIDIA/cosmos-framework" rel="nofollow">cosmos-framework</a>.</p> <p>Mulai dari prompt teks biasa yang singkat dan simpan ke <code>assets/prompt.txt</code>. Untuk contoh teks-ke-video di bawah, prompt aslinya adalah <em>“A robotic arm is cleaning a plate in a kitchen”</em>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Disalin</div><!----></button><!----></div> <pre class="language-bash "><!----><span class="hljs-built_in">mkdir</span> -p assets | |
| <span class="hljs-built_in">echo</span> <span class="hljs-string">"A robotic arm is cleaning a plate in a kitchen"</span> > assets/prompt.txt<!----></pre></div><!----> <p>Lalu, instal framework dan jalankan upsampler. Contoh di bawah ini melakukan upsampling untuk text-to-video menggunakan Opus-4.6:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Disalin</div><!----></button><!----></div> <pre class="language-bash "><!---->git <span class="hljs-built_in">clone</span> https://github.com/NVIDIA/cosmos-framework.git packages/cosmos-framework | |
| pip install -e packages/cosmos-framework | |
| <span class="hljs-built_in">export</span> PROMPT_UPSAMPLER_ENDPOINT_URL=<span class="hljs-string">"https://api.anthropic.com/v1/"</span> | |
| <span class="hljs-built_in">export</span> PROMPT_UPSAMPLER_MODEL_NAME=<span class="hljs-string">"claude-opus-4-6"</span> | |
| <span class="hljs-built_in">export</span> PROMPT_UPSAMPLER_API_TOKEN=<span class="hljs-string">"<your_token>"</span> | |
| python -m cosmos_framework.inference.prompt_upsampling \ | |
| --input assets/prompt.txt \ | |
| --output assets/example_t2v_prompt.json \ | |
| --mode text2video \ | |
| --endpoint-url <span class="hljs-string">"<span class="hljs-variable">${PROMPT_UPSAMPLER_ENDPOINT_URL}</span>"</span> \ | |
| --model <span class="hljs-string">"<span class="hljs-variable">${PROMPT_UPSAMPLER_MODEL_NAME}</span>"</span> \ | |
| --api-token <span class="hljs-string">"<span class="hljs-variable">${PROMPT_UPSAMPLER_API_TOKEN}</span>"</span> \ | |
| --resolution 720 \ | |
| --aspect-ratio <span class="hljs-string">"16,9"</span><!----></pre></div><!----> <p>Ubah <code>--mode</code> agar sesuai dengan alur kerja yang Anda tuju (<code>text2image</code>, <code>text2video</code>, <code>image2video</code>). Perintah ini menulis prompt yang di-upsample ke file <code>--output</code> sebagai array JSON (satu objek per baris non-kosong di <code>--input</code>); berikan path <code>.jsonl</code> sebagai gantinya untuk mendapatkan satu objek JSON per baris. Untuk <code>image2video</code>, Anda juga harus menyediakan gambar kondisioning melalui <code>--image-url</code> (URL atau path lokal) atau <code>--image-list</code> (satu gambar per prompt).</p> <p>Prompt positif yang sudah di-upsample sebelumnya (<code>assets/example_t2v_prompt.json</code>) dan prompt negatif (<code>assets/negative_prompt.json</code>) disediakan untuk kemudahan, dan digunakan oleh contoh generasi di bawah. Contoh memuat file JSON ini dan meneruskannya ke pipeline sebagai string JSON melalui <code>json.dumps(...)</code>.</p> <!--[1--><h2 class="relative group"><a id="text-to-video" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#text-to-video"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Teks-ke-video<!----></span></h2><!--]--><!----> <p>Generasi multi-frame yang dikondisikan hanya pada teks. Pilih <code>num_frames</code> berdasarkan durasi target — nilai default <code>num_frames=189</code> menghasilkan ≈ 7,9 detik pada 24 FPS. Prompt dan negative prompt dibaca dari file yang di-upsample menggunakan JSON, seperti yang dijelaskan dalam <a href="#prompt-upsampling">Prompt upsampling</a>.</p> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"><!--[--><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd border-gray-800 bg-black dark:bg-gray-700 text-white">Nano</div><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd text-gray-500 cursor-pointer opacity-90 hover:text-gray-700 dark:hover:text-gray-200 hover:shadow-sm">Keren</div><!--]--></div> <div class="language-select"><div><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Disalin</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniPipeline | |
| <span class="hljs-keyword">from</span> diffusers.schedulers.scheduling_unipc_multistep <span class="hljs-keyword">import</span> UniPCMultistepScheduler | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video | |
| <span class="hljs-comment"># JSON-upsampled positive and negative prompts (see "Prompt upsampling" above).</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_t2v_prompt.json"</span>)) | |
| negative_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/negative_prompt.json"</span>)) | |
| pipe = Cosmos3OmniPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Nano"</span>, dtype=torch.bfloat16, device_map=<span class="hljs-string">"cuda"</span> <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| ) | |
| pipe.scheduler = UniPCMultistepScheduler.from_config( | |
| pipe.scheduler.config, flow_shift=<span class="hljs-number">10.0</span>, use_karras_sigmas=<span class="hljs-literal">False</span> | |
| ) | |
| result = pipe( | |
| prompt=json.dumps(json_prompt), | |
| negative_prompt=json.dumps(negative_prompt), | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| num_inference_steps=<span class="hljs-number">35</span>, | |
| guidance_scale=<span class="hljs-number">6.0</span>, | |
| fps=<span class="hljs-number">24.0</span>, | |
| ) | |
| <span class="hljs-comment"># macro_block_size=1 allows arbitrary frame sizes (Cosmos3 outputs are not always divisible by 16).</span> | |
| export_to_video(result.video, <span class="hljs-string">"cosmos3_t2v.mp4"</span>, fps=<span class="hljs-number">24</span>, macro_block_size=<span class="hljs-number">1</span>)<!----></pre></div><!----></div><!----> <div hidden=""><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Disalin</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniPipeline | |
| <span class="hljs-keyword">from</span> diffusers.schedulers.scheduling_unipc_multistep <span class="hljs-keyword">import</span> UniPCMultistepScheduler | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video | |
| <span class="hljs-comment"># JSON-upsampled positive and negative prompts (see "Prompt upsampling" above).</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_t2v_prompt.json"</span>)) | |
| negative_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/negative_prompt.json"</span>)) | |
| pipe = Cosmos3OmniPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Super"</span>, dtype=torch.bfloat16, device_map=<span class="hljs-string">"cuda"</span> <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| ) | |
| pipe.scheduler = UniPCMultistepScheduler.from_config( | |
| pipe.scheduler.config, flow_shift=<span class="hljs-number">10.0</span>, use_karras_sigmas=<span class="hljs-literal">False</span> | |
| ) | |
| result = pipe( | |
| prompt=json.dumps(json_prompt), | |
| negative_prompt=json.dumps(negative_prompt), | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| num_inference_steps=<span class="hljs-number">35</span>, | |
| guidance_scale=<span class="hljs-number">6.0</span>, | |
| fps=<span class="hljs-number">24.0</span>, | |
| ) | |
| <span class="hljs-comment"># macro_block_size=1 allows arbitrary frame sizes (Cosmos3 outputs are not always divisible by 16).</span> | |
| export_to_video(result.video, <span class="hljs-string">"cosmos3_t2v.mp4"</span>, fps=<span class="hljs-number">24</span>, macro_block_size=<span class="hljs-number">1</span>)<!----></pre></div><!----></div><!----><!----></div><!----> <!--[1--><h2 class="relative group"><a id="text-to-image" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#text-to-image"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Teks-ke-gambar<!----></span></h2><!--]--><!----> <p>Generasi satu bingkai. Model dikondisikan hanya pada prompt teks; berikan <code>num_frames=1</code>. Upsample dengan <code>--mode text2image</code> untuk menghasilkan prompt JSON.</p> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"><!--[--><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd border-gray-800 bg-black dark:bg-gray-700 text-white">Nano</div><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd text-gray-500 cursor-pointer opacity-90 hover:text-gray-700 dark:hover:text-gray-200 hover:shadow-sm">Keren</div><!--]--></div> <div class="language-select"><div><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Disalin</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniPipeline | |
| <span class="hljs-comment"># JSON-upsampled prompt (see "Prompt upsampling" above).</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_t2i_prompt.json"</span>)) | |
| pipe = Cosmos3OmniPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Nano"</span>, dtype=torch.bfloat16, device_map=<span class="hljs-string">"cuda"</span> <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| ) | |
| result = pipe(prompt=json.dumps(json_prompt), num_frames=<span class="hljs-number">1</span>, height=<span class="hljs-number">720</span>, width=<span class="hljs-number">1280</span>) | |
| result.video[<span class="hljs-number">0</span>].save(<span class="hljs-string">"cosmos3_t2i.jpg"</span>, <span class="hljs-built_in">format</span>=<span class="hljs-string">"JPEG"</span>, quality=<span class="hljs-number">85</span>)<!----></pre></div><!----></div><!----> <div hidden=""><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Disalin</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniPipeline | |
| <span class="hljs-comment"># JSON-upsampled prompt (see "Prompt upsampling" above).</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_t2i_prompt.json"</span>)) | |
| pipe = Cosmos3OmniPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Super"</span>, dtype=torch.bfloat16, device_map=<span class="hljs-string">"cuda"</span> <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| ) | |
| result = pipe(prompt=json.dumps(json_prompt), num_frames=<span class="hljs-number">1</span>, height=<span class="hljs-number">720</span>, width=<span class="hljs-number">1280</span>) | |
| result.video[<span class="hljs-number">0</span>].save(<span class="hljs-string">"cosmos3_t2i.jpg"</span>, <span class="hljs-built_in">format</span>=<span class="hljs-string">"JPEG"</span>, quality=<span class="hljs-number">85</span>)<!----></pre></div><!----></div><!----><!----></div><!----> <!--[1--><h2 class="relative group"><a id="image-to-video" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#image-to-video"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Gambar-ke-video<!----></span></h2><!--]--><!----> <p>Lewatkan gambar kondisioning melalui <code>image=</code>. Pipeline mengaitkan frame 0 ke gambar yang diberikan dan melakukan denoising pada sisanya. Gambar di-resize sambil mempertahankan rasio aspeknya, dipotong di tengah (center-crop) ke ukuran output yang diminta, dan dinormalisasi dengan pembulatan ekuivalen uint8. Gunakan <code>--mode image2video</code> untuk menghasilkan prompt JSON.</p> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"><!--[--><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd border-gray-800 bg-black dark:bg-gray-700 text-white">Nano</div><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd text-gray-500 cursor-pointer opacity-90 hover:text-gray-700 dark:hover:text-gray-200 hover:shadow-sm">Keren</div><!--]--></div> <div class="language-select"><div><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Disalin</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniPipeline | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video, load_image | |
| <span class="hljs-comment"># JSON-upsampled positive and negative prompts (see "Prompt upsampling" above).</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_i2v_prompt.json"</span>)) | |
| negative_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/negative_prompt_i2v.json"</span>)) | |
| pipe = Cosmos3OmniPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Nano"</span>, dtype=torch.bfloat16, device_map=<span class="hljs-string">"cuda"</span> <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| ) | |
| image = load_image( | |
| <span class="hljs-string">"https://github.com/nvidia-cosmos/cosmos-dependencies/releases/download/assets/robot_153.jpg"</span> | |
| ) | |
| result = pipe( | |
| prompt=json.dumps(json_prompt), | |
| negative_prompt=json.dumps(negative_prompt), | |
| image=image, | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| fps=<span class="hljs-number">24.0</span>, | |
| ) | |
| <span class="hljs-comment"># macro_block_size=1 allows arbitrary frame sizes (Cosmos3 outputs are not always divisible by 16).</span> | |
| export_to_video(result.video, <span class="hljs-string">"cosmos3_i2v.mp4"</span>, fps=<span class="hljs-number">24</span>, macro_block_size=<span class="hljs-number">1</span>)<!----></pre></div><!----></div><!----> <div hidden=""><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Disalin</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniPipeline | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video, load_image | |
| <span class="hljs-comment"># JSON-upsampled positive and negative prompts (see "Prompt upsampling" above).</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_i2v_prompt.json"</span>)) | |
| negative_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/negative_prompt_i2v.json"</span>)) | |
| pipe = Cosmos3OmniPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Super"</span>, dtype=torch.bfloat16, device_map=<span class="hljs-string">"cuda"</span> <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| ) | |
| image = load_image( | |
| <span class="hljs-string">"https://github.com/nvidia-cosmos/cosmos-dependencies/releases/download/assets/robot_153.jpg"</span> | |
| ) | |
| result = pipe( | |
| prompt=json.dumps(json_prompt), | |
| negative_prompt=json.dumps(negative_prompt), | |
| image=image, | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| fps=<span class="hljs-number">24.0</span>, | |
| ) | |
| <span class="hljs-comment"># macro_block_size=1 allows arbitrary frame sizes (Cosmos3 outputs are not always divisible by 16).</span> | |
| export_to_video(result.video, <span class="hljs-string">"cosmos3_i2v.mp4"</span>, fps=<span class="hljs-number">24</span>, macro_block_size=<span class="hljs-number">1</span>)<!----></pre></div><!----></div><!----><!----></div><!----> <!--[1--><h2 class="relative group"><a id="video-to-video" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#video-to-video"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Video-ke-video<!----></span></h2><!--]--><!----> <p>Lewatkan klip kondisioning melalui <code>video=</code> (misalnya dari <code>load_video</code>). Pipeline mengikat frame laten awal yang diberikan oleh <code>condition_frame_indexes_vision</code> (default <code>[0, 1]</code>) ke klip tersebut dan melakukan denoising pada sisanya. Gunakan <code>condition_video_keep</code> (<code>"first"</code> atau <code>"last"</code>) untuk memilih dari ujung mana klip sumber yang lebih panjang frame kondisioning diambil. Seperti pada mode lainnya, prompt harus mengikuti struktur JSON deskriptif yang dijelaskan dalam <a href="#prompt-upsampling">Prompt upsampling</a>.</p> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"><!--[--><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd border-gray-800 bg-black dark:bg-gray-700 text-white">Nano</div><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd text-gray-500 cursor-pointer opacity-90 hover:text-gray-700 dark:hover:text-gray-200 hover:shadow-sm">Keren</div><!--]--></div> <div class="language-select"><div><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Disalin</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniPipeline | |
| <span class="hljs-keyword">from</span> diffusers.schedulers.scheduling_unipc_multistep <span class="hljs-keyword">import</span> UniPCMultistepScheduler | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video, load_video | |
| <span class="hljs-comment"># JSON-upsampled positive and negative prompts (see "Prompt upsampling" above).</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_v2v_prompt.json"</span>)) | |
| negative_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/negative_prompt_i2v.json"</span>)) | |
| pipe = Cosmos3OmniPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Nano"</span>, dtype=torch.bfloat16, device_map=<span class="hljs-string">"cuda"</span> <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| ) | |
| pipe.scheduler = UniPCMultistepScheduler.from_config( | |
| pipe.scheduler.config, flow_shift=<span class="hljs-number">10.0</span>, use_karras_sigmas=<span class="hljs-literal">False</span> | |
| ) | |
| video = load_video( | |
| <span class="hljs-string">"https://github.com/nvidia-cosmos/cosmos-dependencies/raw/refs/heads/assets/cosmos3/inputs/vision/robot_pouring.mp4"</span> | |
| ) | |
| result = pipe( | |
| prompt=json.dumps(json_prompt), | |
| negative_prompt=json.dumps(negative_prompt), | |
| video=video, | |
| condition_frame_indexes_vision=[<span class="hljs-number">0</span>, <span class="hljs-number">1</span>], | |
| condition_video_keep=<span class="hljs-string">"first"</span>, | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| num_inference_steps=<span class="hljs-number">35</span>, | |
| guidance_scale=<span class="hljs-number">6.0</span>, | |
| fps=<span class="hljs-number">24.0</span>, | |
| ) | |
| <span class="hljs-comment"># macro_block_size=1 allows arbitrary frame sizes (Cosmos3 outputs are not always divisible by 16).</span> | |
| export_to_video(result.video, <span class="hljs-string">"cosmos3_v2v.mp4"</span>, fps=<span class="hljs-number">24</span>, macro_block_size=<span class="hljs-number">1</span>)<!----></pre></div><!----></div><!----> <div hidden=""><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Disalin</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniPipeline | |
| <span class="hljs-keyword">from</span> diffusers.schedulers.scheduling_unipc_multistep <span class="hljs-keyword">import</span> UniPCMultistepScheduler | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video, load_video | |
| <span class="hljs-comment"># JSON-upsampled positive and negative prompts (see "Prompt upsampling" above).</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_v2v_prompt.json"</span>)) | |
| negative_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/negative_prompt_i2v.json"</span>)) | |
| pipe = Cosmos3OmniPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Super"</span>, dtype=torch.bfloat16, device_map=<span class="hljs-string">"cuda"</span> <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| ) | |
| pipe.scheduler = UniPCMultistepScheduler.from_config( | |
| pipe.scheduler.config, flow_shift=<span class="hljs-number">10.0</span>, use_karras_sigmas=<span class="hljs-literal">False</span> | |
| ) | |
| video = load_video( | |
| <span class="hljs-string">"https://github.com/nvidia-cosmos/cosmos-dependencies/raw/refs/heads/assets/cosmos3/inputs/vision/robot_pouring.mp4"</span> | |
| ) | |
| result = pipe( | |
| prompt=json.dumps(json_prompt), | |
| negative_prompt=json.dumps(negative_prompt), | |
| video=video, | |
| condition_frame_indexes_vision=[<span class="hljs-number">0</span>, <span class="hljs-number">1</span>], | |
| condition_video_keep=<span class="hljs-string">"first"</span>, | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| num_inference_steps=<span class="hljs-number">35</span>, | |
| guidance_scale=<span class="hljs-number">6.0</span>, | |
| fps=<span class="hljs-number">24.0</span>, | |
| ) | |
| <span class="hljs-comment"># macro_block_size=1 allows arbitrary frame sizes (Cosmos3 outputs are not always divisible by 16).</span> | |
| export_to_video(result.video, <span class="hljs-string">"cosmos3_v2v.mp4"</span>, fps=<span class="hljs-number">24</span>, macro_block_size=<span class="hljs-number">1</span>)<!----></pre></div><!----></div><!----><!----></div><!----> <!--[1--><h2 class="relative group"><a id="video-to-video-with-sound" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#video-to-video-with-sound"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Video-ke-video dengan suara<!----></span></h2><!--]--><!----> <p>Ketika checkpoint membawa <code>sound_tokenizer</code>, tambahkan <code>enable_sound=True</code> ke panggilan video-ke-video untuk secara bersama-sama menghasilkan trek audio yang tersinkronisasi. Gelombang suara dikembalikan bersama video dan dapat dicampur ke dalam MP4 dengan <a href="/docs/diffusers/main/id/api/utilities#diffusers.utils.encode_video">encode_video()</a>.</p> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"><!--[--><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd border-gray-800 bg-black dark:bg-gray-700 text-white">Nano</div><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd text-gray-500 cursor-pointer opacity-90 hover:text-gray-700 dark:hover:text-gray-200 hover:shadow-sm">Keren</div><!--]--></div> <div class="language-select"><div><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Disalin</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniPipeline | |
| <span class="hljs-keyword">from</span> diffusers.schedulers.scheduling_unipc_multistep <span class="hljs-keyword">import</span> UniPCMultistepScheduler | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> encode_video, load_video | |
| <span class="hljs-comment"># JSON-upsampled positive and negative prompts (see "Prompt upsampling" above).</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_v2v_prompt.json"</span>)) | |
| negative_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/negative_prompt_i2v.json"</span>)) | |
| pipe = Cosmos3OmniPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Nano"</span>, dtype=torch.bfloat16, device_map=<span class="hljs-string">"cuda"</span> <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| ) | |
| pipe.scheduler = UniPCMultistepScheduler.from_config( | |
| pipe.scheduler.config, flow_shift=<span class="hljs-number">10.0</span>, use_karras_sigmas=<span class="hljs-literal">False</span> | |
| ) | |
| video = load_video( | |
| <span class="hljs-string">"https://github.com/nvidia-cosmos/cosmos-dependencies/raw/refs/heads/assets/cosmos3/inputs/vision/robot_pouring.mp4"</span> | |
| ) | |
| result = pipe( | |
| prompt=json.dumps(json_prompt), | |
| negative_prompt=json.dumps(negative_prompt), | |
| video=video, | |
| condition_frame_indexes_vision=[<span class="hljs-number">0</span>, <span class="hljs-number">1</span>], | |
| condition_video_keep=<span class="hljs-string">"first"</span>, | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| fps=<span class="hljs-number">24.0</span>, | |
| enable_sound=<span class="hljs-literal">True</span>, | |
| ) | |
| encode_video( | |
| result.video, | |
| fps=<span class="hljs-number">24</span>, | |
| audio=result.sound, | |
| audio_sample_rate=pipe.sound_tokenizer.config.sampling_rate, | |
| output_path=<span class="hljs-string">"cosmos3_v2v_with_sound.mp4"</span>, | |
| )<!----></pre></div><!----></div><!----> <div hidden=""><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Disalin</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniPipeline | |
| <span class="hljs-keyword">from</span> diffusers.schedulers.scheduling_unipc_multistep <span class="hljs-keyword">import</span> UniPCMultistepScheduler | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> encode_video, load_video | |
| <span class="hljs-comment"># JSON-upsampled positive and negative prompts (see "Prompt upsampling" above).</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_v2v_prompt.json"</span>)) | |
| negative_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/negative_prompt_i2v.json"</span>)) | |
| pipe = Cosmos3OmniPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Super"</span>, dtype=torch.bfloat16, device_map=<span class="hljs-string">"cuda"</span> <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| ) | |
| pipe.scheduler = UniPCMultistepScheduler.from_config( | |
| pipe.scheduler.config, flow_shift=<span class="hljs-number">10.0</span>, use_karras_sigmas=<span class="hljs-literal">False</span> | |
| ) | |
| video = load_video( | |
| <span class="hljs-string">"https://github.com/nvidia-cosmos/cosmos-dependencies/raw/refs/heads/assets/cosmos3/inputs/vision/robot_pouring.mp4"</span> | |
| ) | |
| result = pipe( | |
| prompt=json.dumps(json_prompt), | |
| negative_prompt=json.dumps(negative_prompt), | |
| video=video, | |
| condition_frame_indexes_vision=[<span class="hljs-number">0</span>, <span class="hljs-number">1</span>], | |
| condition_video_keep=<span class="hljs-string">"first"</span>, | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| fps=<span class="hljs-number">24.0</span>, | |
| enable_sound=<span class="hljs-literal">True</span>, | |
| ) | |
| encode_video( | |
| result.video, | |
| fps=<span class="hljs-number">24</span>, | |
| audio=result.sound, | |
| audio_sample_rate=pipe.sound_tokenizer.config.sampling_rate, | |
| output_path=<span class="hljs-string">"cosmos3_v2v_with_sound.mp4"</span>, | |
| )<!----></pre></div><!----></div><!----><!----></div><!----> <!--[1--><h2 class="relative group"><a id="text-to-video-with-sound" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#text-to-video-with-sound"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Teks-ke-video dengan suara<!----></span></h2><!--]--><!----> <p>Jika checkpoint membawa <code>sound_tokenizer</code>, berikan <code>enable_sound=True</code> untuk menghasilkan trek audio yang tersinkronisasi secara bersama. Gelombang suara dikembalikan bersama video dan dapat dicampur ke dalam MP4 dengan <a href="/docs/diffusers/main/id/api/utilities#diffusers.utils.encode_video">encode_video()</a>.</p> <p>Ini adalah pemanggilan yang sama dengan contoh teks-ke-video di atas dengan <code>enable_sound=True</code> yang ditambahkan:</p> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"><!--[--><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd border-gray-800 bg-black dark:bg-gray-700 text-white">Nano</div><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd text-gray-500 cursor-pointer opacity-90 hover:text-gray-700 dark:hover:text-gray-200 hover:shadow-sm">Keren</div><!--]--></div> <div class="language-select"><div><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Disalin</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniPipeline | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> encode_video | |
| <span class="hljs-comment"># JSON-upsampled positive and negative prompts (see "Prompt upsampling" above).</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_t2v_sound_prompt.json"</span>)) | |
| negative_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/negative_prompt.json"</span>)) | |
| pipe = Cosmos3OmniPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Nano"</span>, dtype=torch.bfloat16, device_map=<span class="hljs-string">"cuda"</span> <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| ) | |
| result = pipe( | |
| prompt=json.dumps(json_prompt), | |
| negative_prompt=json.dumps(negative_prompt), | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| fps=<span class="hljs-number">24.0</span>, | |
| enable_sound=<span class="hljs-literal">True</span>, | |
| ) | |
| encode_video( | |
| result.video, | |
| fps=<span class="hljs-number">24</span>, | |
| audio=result.sound, | |
| audio_sample_rate=pipe.sound_tokenizer.config.sampling_rate, | |
| output_path=<span class="hljs-string">"cosmos3_with_sound.mp4"</span>, | |
| )<!----></pre></div><!----></div><!----> <div hidden=""><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Disalin</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniPipeline | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> encode_video | |
| <span class="hljs-comment"># JSON-upsampled positive and negative prompts (see "Prompt upsampling" above).</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_t2v_sound_prompt.json"</span>)) | |
| negative_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/negative_prompt.json"</span>)) | |
| pipe = Cosmos3OmniPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Super"</span>, dtype=torch.bfloat16, device_map=<span class="hljs-string">"cuda"</span> <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| ) | |
| result = pipe( | |
| prompt=json.dumps(json_prompt), | |
| negative_prompt=json.dumps(negative_prompt), | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| fps=<span class="hljs-number">24.0</span>, | |
| enable_sound=<span class="hljs-literal">True</span>, | |
| ) | |
| encode_video( | |
| result.video, | |
| fps=<span class="hljs-number">24</span>, | |
| audio=result.sound, | |
| audio_sample_rate=pipe.sound_tokenizer.config.sampling_rate, | |
| output_path=<span class="hljs-string">"cosmos3_with_sound.mp4"</span>, | |
| )<!----></pre></div><!----></div><!----><!----></div><!----> <!--[1--><h2 class="relative group"><a id="action-conditioned-generation" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#action-conditioned-generation"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Generasi yang dikondisikan oleh aksi<!----></span></h2><!--]--><!----> <p>Action runs mengelompokkan setiap input spesifik aksi ke dalam <a href="/docs/diffusers/main/id/api/pipelines/cosmos3#diffusers.CosmosActionCondition">CosmosActionCondition</a> yang diteruskan melalui argumen <code>action</code> alih-alih argumen tingkat atas <code>image</code> / <code>video</code> / <code>height</code> / <code>width</code>. Atur <code>resolution_tier</code> (<code>256</code>/<code>480</code>/<code>704</code>/<code>720</code>) sedekat mungkin dengan resolusi asli video input; parameter ini memilih kanvas conditioning. Cosmos 3 mendukung tiga mode aksi — <code>policy</code>, <code>forward_dynamics</code>, dan <code>inverse_dynamics</code>. <code>policy</code> dan <code>forward_dynamics</code> hanya melakukan conditioning pada frame pertama (sehingga <code>image</code> maupun <code>video</code> sama-sama berfungsi), sedangkan <code>inverse_dynamics</code> memerlukan <code>video</code>. Video conditioning untuk sebuah action run diatur pada <code>action.video</code> (atau <code>action.image</code>), bukan pada argumen <code>video</code> tingkat atas pipeline.</p> <p>Lewatkan deskripsi tugas biasa sebagai <code>prompt</code> dan pilih kamera dengan <code>action.view_point</code> (default <code>"ego_view"</code>; juga <code>"third_person_view"</code>, <code>"wrist_view"</code>, <code>"concat_view"</code>). Pipeline mengubah ini menjadi caption JSON terstruktur yang digunakan untuk melatih model, sehingga prompt aksi tidak perlu di-upsample oleh LLM.</p> <!--[2--><h3 class="relative group"><a id="action-policy" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#action-policy"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Kebijakan tindakan<!----></span></h3><!--]--><!----> <p>Generasi kebijakan aksi memprediksi token video dan aksi masa depan dari bingkai observasi pertama, prompt teks, dan metadata domain aksi. Contoh di bawah ini menggunakan domain robot Bridge dan menuliskan chunk aksi yang diprediksi ke JSON dalam ruang aksi yang dinormalisasi oleh model.</p> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"><!--[--><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd border-gray-800 bg-black dark:bg-gray-700 text-white">Nano</div><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd text-gray-500 cursor-pointer opacity-90 hover:text-gray-700 dark:hover:text-gray-200 hover:shadow-sm">Keren</div><!--]--></div> <div class="language-select"><div><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Disalin</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniPipeline, CosmosActionCondition | |
| <span class="hljs-keyword">from</span> diffusers.schedulers.scheduling_unipc_multistep <span class="hljs-keyword">import</span> UniPCMultistepScheduler | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video, load_video | |
| pipe = Cosmos3OmniPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Nano"</span>, dtype=torch.bfloat16, device_map=<span class="hljs-string">"cuda"</span> <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| ) | |
| pipe.scheduler = UniPCMultistepScheduler.from_config( | |
| pipe.scheduler.config, flow_shift=<span class="hljs-number">10.0</span>, use_karras_sigmas=<span class="hljs-literal">False</span> | |
| ) | |
| prompt = <span class="hljs-string">"Put the pot to the left of the purple item."</span> | |
| video = load_video( | |
| <span class="hljs-string">"https://github.com/nvidia-cosmos/cosmos-dependencies/raw/refs/heads/assets/cosmos3/inputs/action/bridge_20260501_0.mp4"</span> | |
| ) | |
| result = pipe( | |
| prompt=prompt, | |
| action=CosmosActionCondition( | |
| mode=<span class="hljs-string">"policy"</span>, | |
| chunk_size=<span class="hljs-number">16</span>, | |
| domain_name=<span class="hljs-string">"bridge_orig_lerobot"</span>, | |
| resolution_tier=<span class="hljs-number">480</span>, | |
| video=video, | |
| view_point=<span class="hljs-string">"ego_view"</span>, | |
| ), | |
| fps=<span class="hljs-number">5</span>, | |
| num_inference_steps=<span class="hljs-number">30</span>, | |
| guidance_scale=<span class="hljs-number">1.0</span>, | |
| use_system_prompt=<span class="hljs-literal">False</span>, | |
| ) | |
| <span class="hljs-comment"># macro_block_size=1 allows arbitrary frame sizes (Cosmos3 outputs are not always divisible by 16).</span> | |
| export_to_video(result.video, <span class="hljs-string">"sample.mp4"</span>, fps=<span class="hljs-number">5</span>, macro_block_size=<span class="hljs-number">1</span>) | |
| <span class="hljs-keyword">if</span> result.action <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> <span class="hljs-literal">None</span>: | |
| <span class="hljs-keyword">with</span> <span class="hljs-built_in">open</span>(<span class="hljs-string">"sample_action.json"</span>, <span class="hljs-string">"w"</span>) <span class="hljs-keyword">as</span> f: | |
| json.dump(result.action[<span class="hljs-number">0</span>].tolist(), f)<!----></pre></div><!----></div><!----> <div hidden=""><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Disalin</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniPipeline, CosmosActionCondition | |
| <span class="hljs-keyword">from</span> diffusers.schedulers.scheduling_unipc_multistep <span class="hljs-keyword">import</span> UniPCMultistepScheduler | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video, load_video | |
| pipe = Cosmos3OmniPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Super"</span>, dtype=torch.bfloat16, device_map=<span class="hljs-string">"cuda"</span> <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| ) | |
| pipe.scheduler = UniPCMultistepScheduler.from_config( | |
| pipe.scheduler.config, flow_shift=<span class="hljs-number">10.0</span>, use_karras_sigmas=<span class="hljs-literal">False</span> | |
| ) | |
| prompt = <span class="hljs-string">"Put the pot to the left of the purple item."</span> | |
| video = load_video( | |
| <span class="hljs-string">"https://github.com/nvidia-cosmos/cosmos-dependencies/raw/refs/heads/assets/cosmos3/inputs/action/bridge_20260501_0.mp4"</span> | |
| ) | |
| result = pipe( | |
| prompt=prompt, | |
| action=CosmosActionCondition( | |
| mode=<span class="hljs-string">"policy"</span>, | |
| chunk_size=<span class="hljs-number">16</span>, | |
| domain_name=<span class="hljs-string">"bridge_orig_lerobot"</span>, | |
| resolution_tier=<span class="hljs-number">480</span>, | |
| video=video, | |
| view_point=<span class="hljs-string">"ego_view"</span>, | |
| ), | |
| fps=<span class="hljs-number">5</span>, | |
| num_inference_steps=<span class="hljs-number">30</span>, | |
| guidance_scale=<span class="hljs-number">1.0</span>, | |
| use_system_prompt=<span class="hljs-literal">False</span>, | |
| ) | |
| <span class="hljs-comment"># macro_block_size=1 allows arbitrary frame sizes (Cosmos3 outputs are not always divisible by 16).</span> | |
| export_to_video(result.video, <span class="hljs-string">"sample.mp4"</span>, fps=<span class="hljs-number">5</span>, macro_block_size=<span class="hljs-number">1</span>) | |
| <span class="hljs-keyword">if</span> result.action <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> <span class="hljs-literal">None</span>: | |
| <span class="hljs-keyword">with</span> <span class="hljs-built_in">open</span>(<span class="hljs-string">"sample_action.json"</span>, <span class="hljs-string">"w"</span>) <span class="hljs-keyword">as</span> f: | |
| json.dump(result.action[<span class="hljs-number">0</span>].tolist(), f)<!----></pre></div><!----></div><!----><!----></div><!----> <!--[1--><h2 class="relative group"><a id="seacache" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#seacache"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->SeaCache<!----></span></h2><!--]--><!----> <p>SeaCache dinonaktifkan secara default. Cosmos 3 mendukung pengaktifan eksplisit dengan <a href="/docs/diffusers/main/id/api/cache#diffusers.SeaCacheConfig">SeaCacheConfig</a>. SeaCache menggunakan ulang residual transformer ketika indikator Spectral-Evolution-Aware berubah secara lambat, sehingga mengurangi jumlah eksekusi transformer penuh. Indikator ini dihitung dari latens visual mentah, termasuk frame kondisioning bersih untuk generasi gambar-ke-video. Aktifkan pada transformer dengan callback metadata scheduler dari pipeline:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Disalin</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniPipeline, SeaCacheConfig | |
| pipe = Cosmos3OmniPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Nano"</span>, dtype=torch.bfloat16, device_map=<span class="hljs-string">"cuda"</span> | |
| ) | |
| pipe.transformer.enable_cache(SeaCacheConfig(threshold=<span class="hljs-number">0.2</span>, max_consecutive_cached=<span class="hljs-number">2</span>))<!----></pre></div><!----> <p>API tingkat model yang sama berfungsi dengan <a href="/docs/diffusers/main/id/api/pipelines/cosmos3#diffusers.Cosmos3OmniPipeline">Cosmos3OmniPipeline</a>, <a href="/docs/diffusers/main/id/api/pipelines/cosmos3#diffusers.Cosmos3OmniModularPipeline">Cosmos3OmniModularPipeline</a>, dan <a href="/docs/diffusers/main/id/api/pipelines/cosmos3#diffusers.Cosmos3DistilledModularPipeline">Cosmos3DistilledModularPipeline</a>. SeaCache bersifat aproksimasi dan dapat mengubah output yang dihasilkan. Nonaktifkan dengan <code>pipe.transformer.disable_cache()</code> ketika Anda membutuhkan setiap langkah denoising untuk mengeksekusi transformer secara penuh. Status cache direset setelah setiap pemanggilan pipeline, dan cabang panduan kondisional serta non-kondisional mempertahankan riwayat yang independen.</p> <p>Cosmos 3 mempertahankan gerbang SeaCache di luar lapisan dekodernya yang berulang, sehingga kompatibel dengan kompilasi regional. | |
| Kompilasi lapisan setelah mengaktifkan cache:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Disalin</div><!----></button><!----></div> <pre class="language-python "><!---->pipe.transformer.compile_repeated_blocks(fullgraph=<span class="hljs-literal">True</span>)<!----></pre></div><!----> <p>SeaCache juga mendukung helper paralel konteks Ulysses Cosmos 3 dan helper paralel tensor berbasis DTensor yang didokumentasikan di bawah. Keputusan cache disinkronkan antar rank. Kompilasi model penuh dan strategi pemecahan model lainnya tidak diklaim.</p> <!--[1--><h2 class="relative group"><a id="context-parallelism" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#context-parallelism"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Paralelisme konteks<!----></span></h2><!--]--><!----> <p>Untuk video panjang atau resolusi tinggi, satu langkah forward dapat melebihi anggaran memori dan latensi satu GPU. Cosmos 3 mendukung <strong>context parallelism (CP)</strong> untuk membagi dimensi sekuens di antara beberapa GPU, memecah komputasi attention sehingga setiap perangkat hanya menyimpan sebagian token.</p> <p>Cosmos 3 mendukung paralelisme konteks <strong>Ulysses</strong> (pertukaran urutan/kepala all-to-all). Ring attention tidak didukung.</p> <p>Berbeda dengan sebagian besar model diffusers, Cosmos 3 tidak <strong>menyambungkan</strong> CP ke dalam transformer atau jalur deklaratif <code>enable_parallelism()</code>: grouped-query attention-nya, stream pemahaman/generasi yang terpisah (stream generasi memperhatikan keduanya), dan panjang per-stream yang tidak seragam tidak dapat diekspresikan sebagai <code>_cp_plan</code>. Sebagai gantinya, model ini mengekspos seam shard/gather kecil yang no-op, dan implementasinya berada di <a href="https://github.com/huggingface/diffusers/blob/main/examples/cosmos3/cosmos_parallel.py" rel="nofollow"><code>examples/cosmos3/cosmos_parallel.py</code></a> — sebuah modul mandiri yang dapat Anda baca dari awal hingga akhir dan sesuaikan. Modul ini menawarkan dua sumbu sharding yang ortogonal dan dapat dikomposisikan:</p> <table><thead><tr><th>Helper</th><th>Shards</th><th>Digunakan untuk</th></tr></thead><tbody><tr><td><code>enable_cosmos3_context_parallel(transformer, cp_mesh)</code></td><td>urutan (CP / Ulysses)</td><td>latensi pada model yang muat di satu GPU (<code>Nano</code>)</td></tr><tr><td><code>enable_cosmos3_tensor_parallel(transformer, tp_mesh)</code></td><td>weights (TP)</td><td>memasangkan model yang tidak muat di satu GPU (<code>Super</code>)</td></tr></tbody></table> <p>Gunakan salah satu atau keduanya secara bersamaan pada mesh 2-D <code>(tp, cp)</code> (lihat <a href="#fitting-large-models-with-tensor-parallelism">Mengakomodasi model besar dengan tensor parallelism</a>).</p> <p>Dua persyaratan khusus untuk Cosmos 3:</p> <ul><li>Gunakan backend perhatian <code>native</code>. Cosmos 3 menggunakan grouped-query attention (GQA), dan backend SDPA native adalah satu-satunya yang menerima <code>enable_gqa</code> (cuDNN dan flash menolaknya). Helper memperluas kepala KV menjadi jumlah kepala query dan memanggil SDPA dengan <code>enable_gqa=False</code> sehingga tetap di-dispatch ke kernel flash (fallback matematika akan mematerialkan skor penuh <code>[S, S]</code> dan menyebabkan OOM pada urutan panjang).</li> <li>Derajat CP (Ulysses) harus membagi jumlah query-head (32 untuk <code>Nano</code>, 64 untuk <code>Super</code>); untuk TP, derajatnya harus membagi KV heads (8). Aliran pemahaman (teks) dan generasi (video/suara) di-shard secara independen sepanjang sequence, dan panjang yang tidak rata di-zero-pad secara internal menjadi kelipatan dari world size.</li></ul> <!--[2--><h3 class="relative group"><a id="run-it" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#run-it"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Jalankan<!----></span></h3><!--]--><!----> <p>CLI lengkap <a href="https://github.com/huggingface/diffusers/blob/main/examples/cosmos3/inference_cosmos3.py" rel="nofollow"><code>examples/cosmos3/inference_cosmos3.py</code></a> menggunakan <a href="/docs/diffusers/main/id/api/pipelines/cosmos3#diffusers.Cosmos3OmniModularPipeline">Cosmos3OmniModularPipeline</a> dan memanfaatkan kembali helper ini, sehingga <strong>semua modality</strong> (text-to-image/video, image-to-video, sound, action modes) dapat dijalankan multi-GPU melalui <code>--tp-degree</code> / <code>--cp-degree</code>. Jalankan dengan <a href="https://docs.pytorch.org/docs/stable/elastic/run.html" rel="nofollow">torchrun</a>; <code>--tp-degree * --cp-degree</code> harus sama dengan <code>--nproc_per_node</code>. Setiap rank menghasilkan output yang sama; rank 0 yang menuliskannya.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Disalin</div><!----></button><!----></div> <pre class="language-bash "><!----><span class="hljs-comment"># CP only — Nano (fits one GPU); CP degree must divide 32 query heads.</span> | |
| torchrun --nproc_per_node=4 examples/cosmos3/inference_cosmos3.py --model nano --cp-degree 4 --prompt <span class="hljs-string">"..."</span> | |
| <span class="hljs-comment"># TP only — Super; TP degree must divide 64 query heads and 8 KV heads.</span> | |
| torchrun --nproc_per_node=4 examples/cosmos3/inference_cosmos3.py --model super --tp-degree 4 --prompt <span class="hljs-string">"..."</span> | |
| <span class="hljs-comment"># TP + CP — Super, with sound (TP=2 x CP=2 across 4 GPUs).</span> | |
| torchrun --nproc_per_node=4 examples/cosmos3/inference_cosmos3.py \ | |
| --model super --tp-degree 2 --cp-degree 2 --enable-sound --prompt <span class="hljs-string">"..."</span><!----></pre></div><!----> <p>Bobot <code>Super</code> yang berukuran ~120 GB tidak muat di satu GPU 96 GB, sehingga memerlukan TP; <code>Nano</code> muat di satu GPU, sehingga CP untuknya murni sebagai optimasi latensi. (Kedua flag diabaikan untuk menjalankan single-GPU.)</p> <!--[2--><h3 class="relative group"><a id="fitting-large-models-with-tensor-parallelism" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#fitting-large-models-with-tensor-parallelism"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Melatih model besar dengan paralelisme tensor<!----></span></h3><!--]--><!----> <p>CP memecah <em>aktivasi</em> tetapi mereplikasi setiap bobot di setiap rank, sehingga tidak mengurangi jejak bobot model — model yang tidak muat di satu GPU tetap tidak akan muat hanya dengan CP. Untuk memecah <strong>bobot</strong>, <code>enable_cosmos3_tensor_parallel(transformer, tp_mesh)</code> menerapkan tensor parallelism gaya Megatron pada sumbu mesh kedua yang ortogonal:</p> <ul><li>Proyeksi attention dan MLP di-shard per kolom/baris di seluruh grup TP (<code>to_q/to_k/to_v</code> + <code>add_q/k/v</code> dan <code>gate/up</code> dari MLP bersifat paralel kolom; <code>to_out/to_add_out</code> dan <code>down</code> dari MLP bersifat paralel baris dengan all-reduce). Setiap rank pada akhirnya memiliki <code>query_heads / tp</code> kepala query dan <code>kv_heads / tp</code> kepala KV.</li> <li>TP dikomposisikan dengan CP pada mesh perangkat 2-D <code>(tp, cp)</code>: TP membagi head/berat secara persisten, sedangkan CP memotong (shard) urutan di atasnya. Kendalanya adalah <code>tp</code> harus membagi habis KV heads (8), dan <code>tp * cp</code> harus membagi habis query heads (32 untuk <code>Nano</code>, 64 untuk <code>Super</code>).</li> <li>Bobot dimuat ke CPU dan dipartisi ke GPU secara per lapisan, sehingga model lengkap tidak pernah direalisasikan pada satu perangkat.</li></ul> <blockquote class="tip"><p>TP menerbitkan all-reduce pada setiap blok attention dan MLP, sehingga sangat berat dari segi bandwidth. Pada host tanpa NVLink, ini menjadi biaya yang dominan; utamakan derajat TP terkecil yang memungkinkan bobot muat, dan masukkan GPU yang tersisa ke CP.</p></blockquote> <!--[2--><h3 class="relative group"><a id="use-it-in-your-own-modular-pipeline" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#use-it-in-your-own-modular-pipeline"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Gunakan di pipeline modular Anda sendiri<!----></span></h3><!--]--><!----> <p>Flag CLI sangat praktis, tetapi Anda dapat memanggil helper secara langsung dengan <a href="/docs/diffusers/main/id/api/pipelines/cosmos3#diffusers.Cosmos3OmniModularPipeline">Cosmos3OmniModularPipeline</a>. Muat konfigurasi pipeline dan komponen di CPU, terapkan TP <em>sebelum</em> memindahkan pipeline ke GPU lokal rank, beralih ke backend <code>native</code>, dan kemudian aktifkan CP. Jangan gunakan <code>device_map</code> untuk alur ini:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Disalin</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> os | |
| <span class="hljs-keyword">import</span> sys | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">import</span> torch.distributed <span class="hljs-keyword">as</span> dist | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniModularPipeline | |
| <span class="hljs-keyword">from</span> diffusers.schedulers.scheduling_unipc_multistep <span class="hljs-keyword">import</span> UniPCMultistepScheduler | |
| <span class="hljs-keyword">from</span> torch.distributed.device_mesh <span class="hljs-keyword">import</span> init_device_mesh | |
| <span class="hljs-comment"># Make the helper module importable.</span> | |
| sys.path.insert(<span class="hljs-number">0</span>, <span class="hljs-string">"examples/cosmos3"</span>) | |
| <span class="hljs-keyword">from</span> cosmos_parallel <span class="hljs-keyword">import</span> ( | |
| enable_cosmos3_context_parallel, | |
| enable_cosmos3_flash_attention, | |
| enable_cosmos3_tensor_parallel, | |
| ) | |
| <span class="hljs-comment"># torchrun sets RANK / WORLD_SIZE / LOCAL_RANK. Pick tp_degree * cp_degree == world size.</span> | |
| local_rank = <span class="hljs-built_in">int</span>(os.environ[<span class="hljs-string">"LOCAL_RANK"</span>]) | |
| torch.cuda.set_device(local_rank) | |
| dist.init_process_group(<span class="hljs-string">"nccl"</span>) | |
| mesh = init_device_mesh(<span class="hljs-string">"cuda"</span>, (tp_degree, cp_degree), mesh_dim_names=(<span class="hljs-string">"tp"</span>, <span class="hljs-string">"cp"</span>)) | |
| <span class="hljs-comment"># Load components on CPU first; a TP-sharded model may not fit one GPU.</span> | |
| pipe = Cosmos3OmniModularPipeline.from_pretrained(model_id) | |
| pipe.load_components(dtype=torch.bfloat16) | |
| pipe.enable_safety_checker() | |
| <span class="hljs-keyword">if</span> tp_degree > <span class="hljs-number">1</span>: | |
| enable_cosmos3_tensor_parallel(pipe.transformer, mesh[<span class="hljs-string">"tp"</span>]) <span class="hljs-comment"># shard weights -> GPUs</span> | |
| pipe.to(<span class="hljs-string">f"cuda:<span class="hljs-subst">{local_rank}</span>"</span>) <span class="hljs-comment"># move the replicated remainder</span> | |
| pipe.transformer.set_attention_backend(<span class="hljs-string">"native"</span>) | |
| <span class="hljs-keyword">if</span> cp_degree > <span class="hljs-number">1</span>: | |
| enable_cosmos3_context_parallel(pipe.transformer, mesh[<span class="hljs-string">"cp"</span>]) <span class="hljs-comment"># shard the sequence</span> | |
| <span class="hljs-keyword">elif</span> tp_degree > <span class="hljs-number">1</span>: | |
| enable_cosmos3_flash_attention(pipe.transformer) <span class="hljs-comment"># GQA-safe dense attention</span> | |
| <span class="hljs-comment"># Modular pipelines replace components through update_components().</span> | |
| scheduler = UniPCMultistepScheduler.from_config( | |
| pipe.scheduler.config, flow_shift=<span class="hljs-number">10.0</span>, use_karras_sigmas=<span class="hljs-literal">False</span> | |
| ) | |
| pipe.update_components(scheduler=scheduler) | |
| <span class="hljs-comment"># A single output name returns that value; a list returns a dictionary.</span> | |
| outputs = pipe( | |
| prompt=<span class="hljs-string">'{"scene":"A robot arm in a kitchen"}'</span>, | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| output=[<span class="hljs-string">"videos"</span>, <span class="hljs-string">"sound"</span>, <span class="hljs-string">"sampling_rate"</span>, <span class="hljs-string">"action"</span>], | |
| ) | |
| videos = outputs[<span class="hljs-string">"videos"</span>] | |
| sound = outputs[<span class="hljs-string">"sound"</span>] <span class="hljs-comment"># None unless sound generation was requested.</span> | |
| action = outputs[<span class="hljs-string">"action"</span>] <span class="hljs-comment"># None unless an action workflow produced actions.</span><!----></pre></div><!----> <p>Untuk CP saja (tanpa sharding bobot), gunakan mesh 1-D: <code>init_device_mesh("cuda", (world_size,), mesh_dim_names=("cp",))</code> dan cukup <code>enable_cosmos3_context_parallel</code>.</p> <p><code>enable_safety_checker()</code> memuat dan mengaktifkan checker default; <code>disable_safety_checker()</code> secara eksplisit menonaktifkannya. Gunakan metode pipeline tersebut alih-alih argumen konstruksi task-pipeline <code>enable_safety_checker=</code> atau argumen pemanggilan <code>enable_safety_check=</code>. Pipeline modular juga tidak mengembalikan <code>Cosmos3OmniPipelineOutput</code>: gunakan <code>output="videos"</code> untuk frame saja, atau daftar output dan kamus yang dikembalikannya seperti yang ditunjukkan di atas, alih-alih <code>result.video</code>, <code>result.sound</code>, atau <code>result.action</code>.</p> <blockquote class="tip"><p>Pada beberapa topologi multi-GPU, NCCL all-to-all pertama dapat hang. Jika eksekusi CP macet di awal langkah denoising pertama, setel <code>NCCL_P2P_DISABLE=1</code> di lingkungan sebelum meluncurkan <code>torchrun</code>.</p></blockquote> <p>CP dan TP dapat dikombinasikan dengan semua alur kerja di atas (teks-ke-video, gambar-ke-video, teks-ke-video dengan suara, dan generasi yang dikondisikan oleh aksi) serta dengan checkpoint <code>Nano</code> dan <code>Super</code> — hanya baris konstruksi pipeline dan pengaturan paralelisme yang berubah.</p> <!--[1--><h2 class="relative group"><a id="metadata-templates" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#metadata-templates"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Template metadata<!----></span></h2><!--]--><!----> <p><code>tokenize_prompt</code> menambahkan kalimat metadata singkat di dalam pesan pengguna agar LLM melihat kondisi yang digunakan saat model dilatih. Prompt positif menerima kalimat seperti <em>“Video ini berdurasi 7,9 detik dan memiliki 24 FPS.”</em> dan <em>“Video ini memiliki resolusi 720x1280.”</em>; prompt negatif menerima kebalikannya (<em>“… tidak …”</em>).</p> <p>Keduanya aktif secara default. Nonaktifkan salah satu pasangan melalui <code>__call__</code>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Disalin</div><!----></button><!----></div> <pre class="language-python "><!---->result = pipe( | |
| prompt=prompt, | |
| negative_prompt=negative_prompt, | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| fps=<span class="hljs-number">24.0</span>, | |
| add_duration_template=<span class="hljs-literal">False</span>, <span class="hljs-comment"># skip the duration sentence on both prompts</span> | |
| add_resolution_template=<span class="hljs-literal">False</span>, <span class="hljs-comment"># skip the resolution sentence on both prompts</span> | |
| )<!----></pre></div><!----> <p><code>add_duration_template</code> tidak berpengaruh ketika <code>num_frames == 1</code> (mode gambar); dalam kasus tersebut, hanya kalimat resolusi yang ditambahkan.</p> <!--[1--><h2 class="relative group"><a id="safety-checker" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#safety-checker"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Pemeriksa keamanan<!----></span></h2><!--]--><!----> <p>Cosmos3 mengintegrasikan <a href="https://pypi.org/project/cosmos-guardrail/" rel="nofollow"><code>cosmos_guardrail</code></a> <code>CosmosSafetyChecker</code> dan menjalankannya <strong>secara default</strong>. Guardrail teks menolak prompt yang tidak aman sebelum generasi (<code>ValueError</code>); guardrail video dijalankan pada frame yang didekode dan akan memikselasi wajah yang terdeteksi atau menolak output. Output audio tidak dijaga oleh guardrail.</p> <p>Instal dependensi opsional untuk mengaktifkan checker default:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Disalin</div><!----></button><!----></div> <pre class=" "><!---->pip <span class="hljs-keyword">install</span> cosmos_guardrail<!----></pre></div><!----> <p>Checker bersifat wajib di bawah Perjanjian Lisensi Model Terbuka NVIDIA. Dua flag di bawah ini ada untuk alur kerja pengujian dan pengembangan di mana guardrail akan menjadi redundan (misalnya, input sudah dibersihkan, atau Anda sengaja menguji pipeline pada input tepi).</p> <p><strong>Nonaktifkan saat konstruksi</strong> (tidak ada checker yang diinstansiasi, sehingga tidak ada model guardrail yang diunduh atau dimuat ke memori):</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Disalin</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniPipeline | |
| pipe = Cosmos3OmniPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Nano"</span>, | |
| dtype=torch.bfloat16, | |
| device_map=<span class="hljs-string">"cuda"</span>, <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| enable_safety_checker=<span class="hljs-literal">False</span>, | |
| )<!----></pre></div><!----> <p><strong>Nonaktifkan untuk satu panggilan</strong> (pemeriksa tetap dimuat — berguna untuk bypass sekali pakai sambil mempertahankan default aktif untuk panggilan berikutnya):</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Disalin</div><!----></button><!----></div> <pre class="language-python "><!---->result = pipe( | |
| prompt=prompt, | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| fps=<span class="hljs-number">24.0</span>, | |
| enable_safety_check=<span class="hljs-literal">False</span>, | |
| )<!----></pre></div><!----> <p>Untuk menyediakan checker kustom (misalnya, subclass no-op untuk pengujian cepat), lewati sebagai <code>safety_checker=</code>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Disalin</div><!----></button><!----></div> <pre class="language-python "><!---->pipe = Cosmos3OmniPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Nano"</span>, | |
| dtype=torch.bfloat16, | |
| device_map=<span class="hljs-string">"cuda"</span>, <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| safety_checker=MyCustomSafetyChecker(), | |
| )<!----></pre></div><!----> <!--[1--><h2 class="relative group"><a id="diffusers.Cosmos3OmniPipeline" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#diffusers.Cosmos3OmniPipeline"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Cosmos3OmniPipeline<!----></span></h2><!--]--><!----> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><div><span class="group flex space-x-1.5 items-center text-gray-800 bg-gradient-to-r rounded-tr-lg -mt-4 -ml-4 pt-3 px-2.5" id="diffusers.Cosmos3OmniPipeline"><!----><h3 class="!m-0"><span class="flex-1 break-all md:text-lg bg-gradient-to-r px-2.5 py-1.5 rounded-xl from-indigo-50/70 to-white dark:from-gray-900 dark:to-gray-950 dark:text-indigo-300 text-indigo-700"><svg class="mr-1.5 text-indigo-500 inline-block -mt-0.5" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width=".8em" height=".8em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 24 24"><path class="uim-quaternary" d="M20.23 7.24L12 12L3.77 7.24a1.98 1.98 0 0 1 .7-.71L11 2.76c.62-.35 1.38-.35 2 0l6.53 3.77c.29.173.531.418.7.71z" opacity=".25" fill="currentColor"></path><path class="uim-tertiary" d="M12 12v9.5a2.09 2.09 0 0 1-.91-.21L4.5 17.48a2.003 2.003 0 0 1-1-1.73v-7.5a2.06 2.06 0 0 1 .27-1.01L12 12z" opacity=".5" fill="currentColor"></path><path class="uim-primary" d="M20.5 8.25v7.5a2.003 2.003 0 0 1-1 1.73l-6.62 3.82c-.275.13-.576.198-.88.2V12l8.23-4.76c.175.308.268.656.27 1.01z" fill="currentColor"></path></svg><span class="font-light">kelas</span> <span class="font-medium">diffusers.</span><span class="font-semibold">Cosmos3OmniPipeline</span></span></h3><!----> <a id="diffusers.Cosmos3OmniPipeline" class="header-link invisible with-hover:group-hover:visible pr-2" href="#diffusers.Cosmos3OmniPipeline"><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></a> <!--[0--><a class="!ml-auto !text-gray-400 !no-underline text-sm flex items-center" href="https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/cosmos/pipeline_cosmos3_omni.py#L413" target="_blank"><span><</span> <span class="hidden md:block mx-0.5 hover:!underline">source</span> <span>></span></a><!--]--></span> <!--[0--><p class="font-mono text-xs md:text-sm !leading-relaxed !my-6"><span>(</span> <!--[--><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">transformer<span class="opacity-60">: Cosmos3OmniTransformer</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">text_tokenizer<span class="opacity-60">: AutoTokenizer</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">vae<span class="opacity-60">: AutoencoderKLWan</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">scheduler<span class="opacity-60">: UniPCMultistepScheduler</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">sound_tokenizer<span class="opacity-60">: diffusers.models.autoencoders.autoencoder_cosmos3_audio.Cosmos3AVAEAudioTokenizer | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">safety_checker<span class="opacity-60">: diffusers.pipelines.cosmos.pipeline_cosmos3_omni.CosmosSafetyChecker | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">enable_safety_checker<span class="opacity-60">: bool = True</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">default_use_system_prompt<span class="opacity-60">: bool = True</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">use_native_flow_schedule<span class="opacity-60">: bool = False</span></span></span><!--]--> <span>)</span> <!--[-1--><!--]--></p><!--]--> <div class="!mb-10 relative docstring-details "><!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--></div></div><!----> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><div><span class="group flex space-x-1.5 items-center text-gray-800 bg-gradient-to-r rounded-tr-lg -mt-4 -ml-4 pt-3 px-2.5" id="diffusers.Cosmos3OmniPipeline.decode_sound"><!----><h4 class="!m-0"><span class="flex-1 rounded-xl py-0.5 break-all bg-gradient-to-r from-blue-50/60 to-white dark:from-gray-900 dark:to-gray-950 text-blue-700 dark:text-blue-300 font-medium px-2"><svg width="1em" height="1em" viewBox="0 0 32 33" class="mr-1 inline-block -mt-0.5" xmlns="http://www.w3.org/2000/svg"><path d="M5.80566 18.3545C4.90766 17.4565 4.90766 16.0005 5.80566 15.1025L14.3768 6.53142C15.2748 5.63342 16.7307 5.63342 17.6287 6.53142L26.1999 15.1025C27.0979 16.0005 27.0979 17.4565 26.1999 18.3545L17.6287 26.9256C16.7307 27.8236 15.2748 27.8236 14.3768 26.9256L5.80566 18.3545Z" fill="currentColor" fill-opacity="0.25"/><path fill-rule="evenodd" clip-rule="evenodd" d="M16.4801 13.9619C16.4801 12.9761 16.7467 12.5436 16.9443 12.3296C17.1764 12.078 17.5731 11.8517 18.2275 11.707C18.8821 11.5623 19.638 11.5342 20.4038 11.5582C20.7804 11.57 21.1341 11.5932 21.4719 11.6156L21.5263 11.6193C21.8195 11.6389 22.1626 11.6618 22.4429 11.6618V7.40825C22.3209 7.40825 22.1219 7.39596 21.7544 7.37149C21.4202 7.34925 20.9976 7.32115 20.5371 7.30672C19.6286 7.27824 18.4672 7.29779 17.3093 7.55377C16.1512 7.8098 14.8404 8.33724 13.8181 9.4452C12.7612 10.5907 12.2266 12.1236 12.2266 13.9619V15.0127H10.6836V19.2662H12.2266V26.6332H16.4801V19.2662H20.3394V15.0127H16.4801V13.9619Z" fill="currentColor"/></svg>decode_sound</span></h4><!----> <a id="diffusers.Cosmos3OmniPipeline.decode_sound" class="header-link invisible with-hover:group-hover:visible pr-2" href="#diffusers.Cosmos3OmniPipeline.decode_sound"><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></a> <!--[0--><a class="!ml-auto !text-gray-400 !no-underline text-sm flex items-center" href="https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/cosmos/pipeline_cosmos3_omni.py#L517" target="_blank"><span><</span> <span class="hidden md:block mx-0.5 hover:!underline">source</span> <span>></span></a><!--]--></span> <!--[0--><p class="font-mono text-xs md:text-sm !leading-relaxed !my-6"><span>(</span> <!--[--><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">latent<span class="opacity-60">: Tensor</span></span></span><!--]--> <span>)</span> <!--[-1--><!--]--></p><!--]--> <div class="!mb-10 relative docstring-details "><!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--></div></div><!----> <p>Dekode latent suara <code>[C, T]</code> menjadi waveform <code>[audio_ch, N]</code>.</p> <p>Menambahkan/menghapus dimensi batch yang diharapkan oleh decoder tokenizer suara.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><div><span class="group flex space-x-1.5 items-center text-gray-800 bg-gradient-to-r rounded-tr-lg -mt-4 -ml-4 pt-3 px-2.5" id="diffusers.Cosmos3OmniPipeline.prepare_latents"><!----><h4 class="!m-0"><span class="flex-1 rounded-xl py-0.5 break-all bg-gradient-to-r from-blue-50/60 to-white dark:from-gray-900 dark:to-gray-950 text-blue-700 dark:text-blue-300 font-medium px-2"><svg width="1em" height="1em" viewBox="0 0 32 33" class="mr-1 inline-block -mt-0.5" xmlns="http://www.w3.org/2000/svg"><path d="M5.80566 18.3545C4.90766 17.4565 4.90766 16.0005 5.80566 15.1025L14.3768 6.53142C15.2748 5.63342 16.7307 5.63342 17.6287 6.53142L26.1999 15.1025C27.0979 16.0005 27.0979 17.4565 26.1999 18.3545L17.6287 26.9256C16.7307 27.8236 15.2748 27.8236 14.3768 26.9256L5.80566 18.3545Z" fill="currentColor" fill-opacity="0.25"/><path fill-rule="evenodd" clip-rule="evenodd" d="M16.4801 13.9619C16.4801 12.9761 16.7467 12.5436 16.9443 12.3296C17.1764 12.078 17.5731 11.8517 18.2275 11.707C18.8821 11.5623 19.638 11.5342 20.4038 11.5582C20.7804 11.57 21.1341 11.5932 21.4719 11.6156L21.5263 11.6193C21.8195 11.6389 22.1626 11.6618 22.4429 11.6618V7.40825C22.3209 7.40825 22.1219 7.39596 21.7544 7.37149C21.4202 7.34925 20.9976 7.32115 20.5371 7.30672C19.6286 7.27824 18.4672 7.29779 17.3093 7.55377C16.1512 7.8098 14.8404 8.33724 13.8181 9.4452C12.7612 10.5907 12.2266 12.1236 12.2266 13.9619V15.0127H10.6836V19.2662H12.2266V26.6332H16.4801V19.2662H20.3394V15.0127H16.4801V13.9619Z" fill="currentColor"/></svg>prepare_latents</span></h4><!----> <a id="diffusers.Cosmos3OmniPipeline.prepare_latents" class="header-link invisible with-hover:group-hover:visible pr-2" href="#diffusers.Cosmos3OmniPipeline.prepare_latents"><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></a> <!--[0--><a class="!ml-auto !text-gray-400 !no-underline text-sm flex items-center" href="https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/cosmos/pipeline_cosmos3_omni.py#L763" target="_blank"><span><</span> <span class="hidden md:block mx-0.5 hover:!underline">source</span> <span>></span></a><!--]--></span> <!--[0--><p class="font-mono text-xs md:text-sm !leading-relaxed !my-6"><span>(</span> <!--[--><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">image<span class="opacity-60">: typing.Union[PIL.Image.Image, numpy.ndarray, torch.Tensor, NoneType] = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">video<span class="opacity-60">: typing.Union[list[PIL.Image.Image], torch.Tensor, numpy.ndarray, NoneType] = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">condition_frame_indexes_vision<span class="opacity-60">: collections.abc.Iterable[int] = (0, 1)</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">condition_video_keep<span class="opacity-60">: typing.Literal['first', 'last'] = 'first'</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">num_frames<span class="opacity-60">: int | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">height<span class="opacity-60">: int | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">width<span class="opacity-60">: int | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">fps<span class="opacity-60">: float = 24.0</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">latents<span class="opacity-60">: typing.Optional[torch.Tensor] = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">sound_latents<span class="opacity-60">: typing.Optional[torch.Tensor] = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">action_latents<span class="opacity-60">: typing.Optional[torch.Tensor] = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">generator<span class="opacity-60">: typing.Optional[torch.Generator] = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">device<span class="opacity-60">: str = 'cuda'</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">dtype<span class="opacity-60">: dtype = torch.bfloat16</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">enable_sound<span class="opacity-60">: bool = False</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">action<span class="opacity-60">: CosmosActionCondition | None = None</span></span></span><!--]--> <span>)</span> <!--[-1--><!--]--></p><!--]--> <div class="!mb-10 relative docstring-details "><!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--></div></div><!----> <p>Bangun kondisioning + noise awal untuk satu sampel.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><div><span class="group flex space-x-1.5 items-center text-gray-800 bg-gradient-to-r rounded-tr-lg -mt-4 -ml-4 pt-3 px-2.5" id="diffusers.Cosmos3OmniPipeline.tokenize_prompt"><!----><h4 class="!m-0"><span class="flex-1 rounded-xl py-0.5 break-all bg-gradient-to-r from-blue-50/60 to-white dark:from-gray-900 dark:to-gray-950 text-blue-700 dark:text-blue-300 font-medium px-2"><svg width="1em" height="1em" viewBox="0 0 32 33" class="mr-1 inline-block -mt-0.5" xmlns="http://www.w3.org/2000/svg"><path d="M5.80566 18.3545C4.90766 17.4565 4.90766 16.0005 5.80566 15.1025L14.3768 6.53142C15.2748 5.63342 16.7307 5.63342 17.6287 6.53142L26.1999 15.1025C27.0979 16.0005 27.0979 17.4565 26.1999 18.3545L17.6287 26.9256C16.7307 27.8236 15.2748 27.8236 14.3768 26.9256L5.80566 18.3545Z" fill="currentColor" fill-opacity="0.25"/><path fill-rule="evenodd" clip-rule="evenodd" d="M16.4801 13.9619C16.4801 12.9761 16.7467 12.5436 16.9443 12.3296C17.1764 12.078 17.5731 11.8517 18.2275 11.707C18.8821 11.5623 19.638 11.5342 20.4038 11.5582C20.7804 11.57 21.1341 11.5932 21.4719 11.6156L21.5263 11.6193C21.8195 11.6389 22.1626 11.6618 22.4429 11.6618V7.40825C22.3209 7.40825 22.1219 7.39596 21.7544 7.37149C21.4202 7.34925 20.9976 7.32115 20.5371 7.30672C19.6286 7.27824 18.4672 7.29779 17.3093 7.55377C16.1512 7.8098 14.8404 8.33724 13.8181 9.4452C12.7612 10.5907 12.2266 12.1236 12.2266 13.9619V15.0127H10.6836V19.2662H12.2266V26.6332H16.4801V19.2662H20.3394V15.0127H16.4801V13.9619Z" fill="currentColor"/></svg>tokenize_prompt</span></h4><!----> <a id="diffusers.Cosmos3OmniPipeline.tokenize_prompt" class="header-link invisible with-hover:group-hover:visible pr-2" href="#diffusers.Cosmos3OmniPipeline.tokenize_prompt"><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></a> <!--[0--><a class="!ml-auto !text-gray-400 !no-underline text-sm flex items-center" href="https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/cosmos/pipeline_cosmos3_omni.py#L1132" target="_blank"><span><</span> <span class="hidden md:block mx-0.5 hover:!underline">source</span> <span>></span></a><!--]--></span> <!--[0--><p class="font-mono text-xs md:text-sm !leading-relaxed !my-6"><span>(</span> <!--[--><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">prompt<span class="opacity-60">: str</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">negative_prompt<span class="opacity-60">: str | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">num_frames<span class="opacity-60">: int = 189</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">height<span class="opacity-60">: int = 720</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">width<span class="opacity-60">: int = 1280</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">fps<span class="opacity-60">: float = 24.0</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">use_system_prompt<span class="opacity-60">: bool | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">add_resolution_template<span class="opacity-60">: bool = True</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">add_duration_template<span class="opacity-60">: bool = True</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">action_mode<span class="opacity-60">: str | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">action_view_point<span class="opacity-60">: str | None = None</span></span></span><!--]--> <span>)</span> <!--[-1--><!--]--></p><!--]--> <div class="!mb-10 relative docstring-details "><!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--></div></div><!----> <p>Terapkan template prompt-augmentation dan lakukan tokenisasi prompt cond/uncond melalui chat template yang dikonfigurasi.</p> <p>Pipeline ini tidak menjalankan pengkode teks terpisah: transformer Cosmos3 gabungan mengonsumsi ID token mentah bersama dengan token visi (dan opsional, token suara).</p> <p>Ketika <code>negative_prompt</code> adalah <code>None</code>, string kosong digunakan; halaman dokumentasi Cosmos3 mendokumentasikan prompt negatif pengendalian kualitas yang direkomendasikan untuk dilewatkan secara eksplisit untuk text2video / image2video. Template durasi dan resolusi ditambahkan ke prompt, dan template invers ditambahkan ke prompt negatif, ketika diaktifkan.</p> <p>Ketika <code>action_mode</code> disetel, prompt diubah menjadi caption JSON aksi terstruktur yang digunakan untuk melatih model (lihat <code>_build_action_json_prompt</code>), dengan menggunakan <code>action_view_point</code> untuk field framing; template metadata datar dilewati karena JSON sudah membawa duration/fps/resolution/aspect_ratio.</p></div></div> <ul><li>semua</li> <li><strong>call</strong></li></ul> <!--[1--><h2 class="relative group"><a id="cosmos3omnimodularpipeline" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#cosmos3omnimodularpipeline"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Cosmos3OmniModularPipeline<!----></span></h2><!--]--><!----> <p>Cosmos 3 juga tersedia sebagai pipeline Modular Diffusers. Pipeline berbasis tugas <a href="/docs/diffusers/main/id/api/pipelines/cosmos3#diffusers.Cosmos3OmniPipeline">Cosmos3OmniPipeline</a> tetap tersedia; pipeline modular ini berdampingan dengannya dan mencakup mode yang sama (<code>text2image</code>, <code>text2video</code>, <code>image2video</code>, <code>video2video</code>, generasi yang dikondisikan oleh aksi, dan <code>transfer</code> (kontrol struktural), dengan opsi suara jika didukung oleh checkpoint).</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Disalin</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniModularPipeline | |
| pipe = Cosmos3OmniModularPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Nano"</span>, dtype=torch.bfloat16 | |
| ) | |
| pipe.load_components(dtype=torch.bfloat16) | |
| pipe.enable_safety_checker() | |
| videos = pipe( | |
| prompt=<span class="hljs-string">'{"scene":"A robot arm in a kitchen"}'</span>, | |
| num_frames=<span class="hljs-number">1</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| output=<span class="hljs-string">"videos"</span>, | |
| ) | |
| <span class="hljs-comment"># Modular pipelines expose declared outputs directly instead of using the task pipeline's</span> | |
| <span class="hljs-comment"># `return_dict`/`Cosmos3OmniPipelineOutput` API.</span> | |
| image = videos[<span class="hljs-number">0</span>]<!----></pre></div><!----> <p>Anda juga dapat memuat melalui <a href="/docs/diffusers/main/id/api/modular_diffusers/pipeline#diffusers.ModularPipeline">ModularPipeline</a> dan membiarkan konfigurasi repositori memilih kelas blok:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Disalin</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> ModularPipeline | |
| pipe = ModularPipeline.from_pretrained(<span class="hljs-string">"nvidia/Cosmos3-Nano"</span>, dtype=torch.bfloat16) | |
| pipe.load_components(dtype=torch.bfloat16) | |
| pipe.enable_safety_checker() | |
| videos = pipe( | |
| prompt=<span class="hljs-string">'{"scene":"A robot arm in a kitchen"}'</span>, num_frames=<span class="hljs-number">1</span>, height=<span class="hljs-number">720</span>, width=<span class="hljs-number">1280</span>, output=<span class="hljs-string">"videos"</span> | |
| )<!----></pre></div><!----> <p>Untuk memeriksa atau menyesuaikan alur kerja modular Cosmos tertentu, gunakan <code>available_workflows</code> + <code>get_workflow()</code>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Disalin</div><!----></button><!----></div> <pre class="language-python "><!---->available = pipe.blocks.available_workflows | |
| image2video_blocks = pipe.blocks.get_workflow(<span class="hljs-string">"image2video"</span>)<!----></pre></div><!----> <!--[2--><h3 class="relative group"><a id="modular-examples-for-all-existing-workflows" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#modular-examples-for-all-existing-workflows"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Contoh modular untuk semua workflow yang ada<!----></span></h3><!--]--><!----> <p>Pipeline modular mendukung tanda panggil yang sama seperti pipeline tugas. Cuplikan di bawah ini mencerminkan setiap contoh generasi yang ditunjukkan di atas (<code>text2video</code>, <code>text2image</code>, <code>image2video</code>, <code>video2video</code>, <code>video2video_sound</code>, <code>text2video_sound</code>, dan <code>action_policy</code>). Transfer (kontrol struktural) memiliki inputnya sendiri dan ditampilkan secara terpisah di <a href="#modular-transfer-structural-control">Transfer modular</a> di bawah ini.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Disalin</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniModularPipeline, CosmosActionCondition | |
| <span class="hljs-keyword">from</span> diffusers.schedulers.scheduling_unipc_multistep <span class="hljs-keyword">import</span> UniPCMultistepScheduler | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> encode_video, export_to_video, load_image, load_video | |
| pipe = Cosmos3OmniModularPipeline.from_pretrained(<span class="hljs-string">"nvidia/Cosmos3-Nano"</span>, dtype=torch.bfloat16) | |
| pipe.load_components(dtype=torch.bfloat16) | |
| pipe.enable_safety_checker() | |
| pipe.to(<span class="hljs-string">"cuda"</span>) <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| pipe.scheduler = UniPCMultistepScheduler.from_config( | |
| pipe.scheduler.config, flow_shift=<span class="hljs-number">10.0</span>, use_karras_sigmas=<span class="hljs-literal">False</span> | |
| ) | |
| <span class="hljs-comment"># text2video</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_t2v_prompt.json"</span>)) | |
| negative_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/negative_prompt.json"</span>)) | |
| videos = pipe( | |
| prompt=json.dumps(json_prompt), | |
| negative_prompt=json.dumps(negative_prompt), | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| num_inference_steps=<span class="hljs-number">35</span>, | |
| guidance_scale=<span class="hljs-number">6.0</span>, | |
| fps=<span class="hljs-number">24.0</span>, | |
| output=<span class="hljs-string">"videos"</span>, | |
| ) | |
| export_to_video(videos, <span class="hljs-string">"cosmos3_modular_t2v.mp4"</span>, fps=<span class="hljs-number">24</span>, macro_block_size=<span class="hljs-number">1</span>) | |
| <span class="hljs-comment"># text2image</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_t2i_prompt.json"</span>)) | |
| videos = pipe(prompt=json.dumps(json_prompt), num_frames=<span class="hljs-number">1</span>, height=<span class="hljs-number">720</span>, width=<span class="hljs-number">1280</span>, output=<span class="hljs-string">"videos"</span>) | |
| videos[<span class="hljs-number">0</span>].save(<span class="hljs-string">"cosmos3_modular_t2i.jpg"</span>, <span class="hljs-built_in">format</span>=<span class="hljs-string">"JPEG"</span>, quality=<span class="hljs-number">85</span>) | |
| <span class="hljs-comment"># image2video</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_i2v_prompt.json"</span>)) | |
| negative_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/negative_prompt_i2v.json"</span>)) | |
| image = load_image(<span class="hljs-string">"https://github.com/nvidia-cosmos/cosmos-dependencies/releases/download/assets/robot_153.jpg"</span>) | |
| videos = pipe( | |
| prompt=json.dumps(json_prompt), | |
| negative_prompt=json.dumps(negative_prompt), | |
| image=image, | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| fps=<span class="hljs-number">24.0</span>, | |
| output=<span class="hljs-string">"videos"</span>, | |
| ) | |
| export_to_video(videos, <span class="hljs-string">"cosmos3_modular_i2v.mp4"</span>, fps=<span class="hljs-number">24</span>, macro_block_size=<span class="hljs-number">1</span>) | |
| <span class="hljs-comment"># video2video</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_v2v_prompt.json"</span>)) | |
| negative_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/negative_prompt_i2v.json"</span>)) | |
| video = load_video( | |
| <span class="hljs-string">"https://github.com/nvidia-cosmos/cosmos-dependencies/raw/refs/heads/assets/cosmos3/inputs/vision/robot_pouring.mp4"</span> | |
| ) | |
| videos = pipe( | |
| prompt=json.dumps(json_prompt), | |
| negative_prompt=json.dumps(negative_prompt), | |
| video=video, | |
| condition_frame_indexes_vision=[<span class="hljs-number">0</span>, <span class="hljs-number">1</span>], | |
| condition_video_keep=<span class="hljs-string">"first"</span>, | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| num_inference_steps=<span class="hljs-number">35</span>, | |
| guidance_scale=<span class="hljs-number">6.0</span>, | |
| fps=<span class="hljs-number">24.0</span>, | |
| output=<span class="hljs-string">"videos"</span>, | |
| ) | |
| export_to_video(videos, <span class="hljs-string">"cosmos3_modular_v2v.mp4"</span>, fps=<span class="hljs-number">24</span>, macro_block_size=<span class="hljs-number">1</span>) | |
| <span class="hljs-comment"># video2video_sound</span> | |
| outputs = pipe( | |
| prompt=json.dumps(json_prompt), | |
| negative_prompt=json.dumps(negative_prompt), | |
| video=video, | |
| condition_frame_indexes_vision=[<span class="hljs-number">0</span>, <span class="hljs-number">1</span>], | |
| condition_video_keep=<span class="hljs-string">"first"</span>, | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| fps=<span class="hljs-number">24.0</span>, | |
| enable_sound=<span class="hljs-literal">True</span>, | |
| output=[<span class="hljs-string">"videos"</span>, <span class="hljs-string">"sound"</span>, <span class="hljs-string">"sampling_rate"</span>], | |
| ) | |
| encode_video( | |
| outputs[<span class="hljs-string">"videos"</span>], | |
| fps=<span class="hljs-number">24</span>, | |
| audio=outputs[<span class="hljs-string">"sound"</span>], | |
| audio_sample_rate=outputs[<span class="hljs-string">"sampling_rate"</span>], | |
| output_path=<span class="hljs-string">"cosmos3_modular_v2v_with_sound.mp4"</span>, | |
| ) | |
| <span class="hljs-comment"># text2video_sound</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_t2v_sound_prompt.json"</span>)) | |
| negative_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/negative_prompt.json"</span>)) | |
| outputs = pipe( | |
| prompt=json.dumps(json_prompt), | |
| negative_prompt=json.dumps(negative_prompt), | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| fps=<span class="hljs-number">24.0</span>, | |
| enable_sound=<span class="hljs-literal">True</span>, | |
| output=[<span class="hljs-string">"videos"</span>, <span class="hljs-string">"sound"</span>, <span class="hljs-string">"sampling_rate"</span>], | |
| ) | |
| encode_video( | |
| outputs[<span class="hljs-string">"videos"</span>], | |
| fps=<span class="hljs-number">24</span>, | |
| audio=outputs[<span class="hljs-string">"sound"</span>], | |
| audio_sample_rate=outputs[<span class="hljs-string">"sampling_rate"</span>], | |
| output_path=<span class="hljs-string">"cosmos3_modular_t2v_with_sound.mp4"</span>, | |
| ) | |
| <span class="hljs-comment"># action_policy</span> | |
| prompt = <span class="hljs-string">"Put the pot to the left of the purple item."</span> | |
| action_video = load_video( | |
| <span class="hljs-string">"https://github.com/nvidia-cosmos/cosmos-dependencies/raw/refs/heads/assets/cosmos3/inputs/action/bridge_20260501_0.mp4"</span> | |
| ) | |
| outputs = pipe( | |
| prompt=prompt, | |
| action=CosmosActionCondition( | |
| mode=<span class="hljs-string">"policy"</span>, | |
| chunk_size=<span class="hljs-number">16</span>, | |
| domain_name=<span class="hljs-string">"bridge_orig_lerobot"</span>, | |
| resolution_tier=<span class="hljs-number">480</span>, | |
| video=action_video, | |
| view_point=<span class="hljs-string">"ego_view"</span>, | |
| ), | |
| fps=<span class="hljs-number">5</span>, | |
| num_inference_steps=<span class="hljs-number">30</span>, | |
| guidance_scale=<span class="hljs-number">1.0</span>, | |
| use_system_prompt=<span class="hljs-literal">False</span>, | |
| output=[<span class="hljs-string">"videos"</span>, <span class="hljs-string">"action"</span>], | |
| ) | |
| export_to_video(outputs[<span class="hljs-string">"videos"</span>], <span class="hljs-string">"cosmos3_modular_action_policy.mp4"</span>, fps=<span class="hljs-number">5</span>, macro_block_size=<span class="hljs-number">1</span>) | |
| <span class="hljs-keyword">if</span> outputs[<span class="hljs-string">"action"</span>] <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> <span class="hljs-literal">None</span>: | |
| <span class="hljs-keyword">with</span> <span class="hljs-built_in">open</span>(<span class="hljs-string">"cosmos3_modular_action_policy.json"</span>, <span class="hljs-string">"w"</span>) <span class="hljs-keyword">as</span> f: | |
| json.dump(outputs[<span class="hljs-string">"action"</span>][<span class="hljs-number">0</span>].tolist(), f)<!----></pre></div><!----> <!--[2--><h3 class="relative group"><a id="modular-transfer-structural-control" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#modular-transfer-structural-control"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Transfer modular (kontrol struktural)<!----></span></h3><!--]--><!----> <p>Transfer mengikuti <strong>video kontrol yang sudah dihitung sebelumnya</strong> (edge, blur, depth, segmentation, atau peta skenario dunia) yang diteruskan melalui <code>control_videos=</code> sebagai pemetaan <code>{hint: video}</code>. Ini hanya untuk video (tanpa <code>image</code> / <code>video</code> / <code>action</code> / <code>enable_sound</code>), prompt-nya adalah caption JSON yang sudah di-upsample sebelumnya (lihat <a href="#prompt-upsampling">Prompt upsampling</a>), dan klip panjang dihasilkan secara autoregresif dalam potongan <code>num_video_frames_per_chunk</code> dan dijahit secara otomatis. <code>guidance_scale</code> adalah CFG teks yang biasa; <code>control_guidance</code> (<code>!= 1.0</code>) secara tambahan memperkuat sinyal kontrol. Nilai awal yang direkomendasikan per hint:</p> <table><thead><tr><th>Petunjuk</th><th><code>guidance_scale</code></th><th><code>control_guidance</code></th><th><code>flow_shift</code></th><th>Geometri</th></tr></thead><tbody><tr><td>Edge / Blur / Kedalaman</td><td>3.0</td><td>1.5</td><td>10.0</td><td>121 frame @ 30 FPS</td></tr><tr><td>Segmentasi</td><td>3.0</td><td>2.0</td><td>10.0</td><td>121 frame @ 30 FPS</td></tr><tr><td>Skenario dunia (WSM)</td><td>1.0</td><td>3.0</td><td>10.0</td><td>101 frame @ 10 FPS</td></tr></tbody></table> <p>Diffusers tidak menyertakan aset kontrol. Aset yang sudah jadi (video kontrol + <code>prompt.json</code> yang sesuai untuk setiap hint, serta <code>negative_prompt.json</code> bersama) tersedia di <a href="https://github.com/NVIDIA/cosmos/tree/6e2c69afafd6444c1ca14a021aba93fcc6b7996a/cookbooks/cosmos3/generator/transfer/assets" rel="nofollow">Cosmos cookbook</a>. Untuk contoh edge di bawah, unduh aset tersebut ke folder lokal <code>assets/</code>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Disalin</div><!----></button><!----></div> <pre class="language-bash "><!---->base=https://github.com/NVIDIA/cosmos/raw/6e2c69afafd6444c1ca14a021aba93fcc6b7996a/cookbooks/cosmos3/generator/transfer/assets | |
| <span class="hljs-built_in">mkdir</span> -p assets/edge | |
| curl -sL <span class="hljs-string">"<span class="hljs-variable">$base</span>/edge/control_edge.mp4"</span> -o assets/edge/control_edge.mp4 | |
| curl -sL <span class="hljs-string">"<span class="hljs-variable">$base</span>/edge/prompt.json"</span> -o assets/edge/prompt.json | |
| curl -sL <span class="hljs-string">"<span class="hljs-variable">$base</span>/negative_prompt.json"</span> -o assets/negative_prompt.json<!----></pre></div><!----> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Disalin</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniModularPipeline | |
| <span class="hljs-keyword">from</span> diffusers.schedulers.scheduling_unipc_multistep <span class="hljs-keyword">import</span> UniPCMultistepScheduler | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video, load_video | |
| pipe = Cosmos3OmniModularPipeline.from_pretrained(<span class="hljs-string">"nvidia/Cosmos3-Nano"</span>, dtype=torch.bfloat16) | |
| pipe.load_components(dtype=torch.bfloat16) | |
| pipe.to(<span class="hljs-string">"cuda"</span>) <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| pipe.scheduler = UniPCMultistepScheduler.from_config( | |
| pipe.scheduler.config, flow_shift=<span class="hljs-number">10.0</span>, use_karras_sigmas=<span class="hljs-literal">False</span> | |
| ) | |
| <span class="hljs-comment"># Downloaded into assets/ from the Cosmos cookbook (see the curl snippet above).</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/edge/prompt.json"</span>)) | |
| negative_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/negative_prompt.json"</span>)) | |
| control_edge = load_video(<span class="hljs-string">"assets/edge/control_edge.mp4"</span>) | |
| videos = pipe( | |
| prompt=json.dumps(json_prompt), | |
| negative_prompt=json.dumps(negative_prompt), | |
| control_videos={<span class="hljs-string">"edge"</span>: control_edge}, | |
| num_frames=<span class="hljs-number">121</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| fps=<span class="hljs-number">30.0</span>, | |
| num_inference_steps=<span class="hljs-number">35</span>, | |
| guidance_scale=<span class="hljs-number">3.0</span>, | |
| control_guidance=<span class="hljs-number">1.5</span>, | |
| output=<span class="hljs-string">"videos"</span>, | |
| ) | |
| export_to_video(videos, <span class="hljs-string">"cosmos3_modular_transfer_edge.mp4"</span>, fps=<span class="hljs-number">30</span>, macro_block_size=<span class="hljs-number">1</span>)<!----></pre></div><!----> <!--[2--><h3 class="relative group"><a id="diffusers.Cosmos3OmniModularPipeline" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#diffusers.Cosmos3OmniModularPipeline"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Teks-ke-gambar dan gambar-ke-video yang telah didistilasi (sedikit langkah)<!----></span></h3><!--]--><!----> <p>Checkpoint distilasi beberapa langkah disajikan oleh <a href="/docs/diffusers/main/id/api/pipelines/cosmos3#diffusers.Cosmos3DistilledModularPipeline">Cosmos3DistilledModularPipeline</a> (blok: <code>Cosmos3DistilledBlocks</code>); <a href="/docs/diffusers/main/id/api/pipelines/cosmos3#diffusers.Cosmos3OmniModularPipeline">Cosmos3OmniModularPipeline</a> dan <a href="/docs/diffusers/main/id/api/pipelines/cosmos3#diffusers.Cosmos3OmniPipeline">Cosmos3OmniPipeline</a> dasar tidak mendukungnya. <code>num_inference_steps</code> dikunci pada panjang konfigurasi pipeline <code>distilled_sigmas</code> (dari <code>modular_model_index.json</code> checkpoint) dan <code>guidance_scale</code> dipaksa menjadi 1.0 karena panduan sudah tertanam dalam bobot — memberikan nilai lain untuk salah satu dari keduanya akan memicu error, dan <code>negative_prompt</code> akan diberi peringatan serta diabaikan.</p> <p>Checkpoint distilasi FP8 (<code>revision="fp8"</code>) tidak mendeklarasikan kebijakan presisi campuran, sehingga setiap langkah tetap menggunakan W8A8 native.</p> <p>Prompts mengikuti struktur JSON deskriptif yang sama dengan model non-distilled, sehingga teks pendek harus di-upsample terlebih dahulu — gunakan <code>--mode text2image</code> (T2I) atau <code>--mode image2video</code> (I2V) seperti yang dijelaskan dalam <a href="#prompt-upsampling">Prompt upsampling</a>, lalu kirimkan JSON melalui <code>json.dumps(...)</code>.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Disalin</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3DistilledModularPipeline | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video, load_image | |
| <span class="hljs-comment"># JSON-upsampled prompt (see "Prompt upsampling" above).</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_t2i_prompt.json"</span>)) | |
| repo = <span class="hljs-string">"nvidia/Cosmos3-Super-Text2Image-4Step"</span> | |
| pipe = Cosmos3DistilledModularPipeline.from_pretrained(repo, dtype=torch.bfloat16) | |
| pipe.load_components(dtype=torch.bfloat16) | |
| pipe.to(<span class="hljs-string">"cuda"</span>) <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| <span class="hljs-comment"># text-to-image (distilled)</span> | |
| videos = pipe( | |
| prompt=json.dumps(json_prompt), | |
| num_frames=<span class="hljs-number">1</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| output=<span class="hljs-string">"videos"</span>, | |
| ) | |
| videos[<span class="hljs-number">0</span>].save(<span class="hljs-string">"cosmos3_distilled_t2i.jpg"</span>, <span class="hljs-built_in">format</span>=<span class="hljs-string">"JPEG"</span>, quality=<span class="hljs-number">85</span>) | |
| <span class="hljs-comment"># image-to-video (distilled) — load the I2V repo instead</span> | |
| <span class="hljs-comment"># JSON-upsampled prompt (see "Prompt upsampling" above); upsampled from the source prompt</span> | |
| <span class="hljs-comment"># "The right robotic hand picks up the red sphere on the shelf."</span> | |
| json_prompt_i2v = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_i2v_prompt.json"</span>)) | |
| repo_i2v = <span class="hljs-string">"nvidia/Cosmos3-Super-Image2Video-4Step"</span> | |
| pipe = Cosmos3DistilledModularPipeline.from_pretrained(repo_i2v, dtype=torch.bfloat16) | |
| pipe.load_components(dtype=torch.bfloat16) | |
| pipe.to(<span class="hljs-string">"cuda"</span>) | |
| image = load_image( | |
| <span class="hljs-string">"https://github.com/nvidia-cosmos/cosmos-dependencies/raw/refs/heads/assets/cosmos3/inputs/vision/robot_153.jpg"</span> | |
| ) | |
| videos = pipe( | |
| prompt=json.dumps(json_prompt_i2v), | |
| image=image, | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| output=<span class="hljs-string">"videos"</span>, | |
| ) | |
| export_to_video(videos, <span class="hljs-string">"cosmos3_distilled_i2v.mp4"</span>, fps=<span class="hljs-number">24</span>, macro_block_size=<span class="hljs-number">1</span>)<!----></pre></div><!----> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><div><span class="group flex space-x-1.5 items-center text-gray-800 bg-gradient-to-r rounded-tr-lg -mt-4 -ml-4 pt-3 px-2.5" id="diffusers.Cosmos3OmniModularPipeline"><!----><h3 class="!m-0"><span class="flex-1 break-all md:text-lg bg-gradient-to-r px-2.5 py-1.5 rounded-xl from-indigo-50/70 to-white dark:from-gray-900 dark:to-gray-950 dark:text-indigo-300 text-indigo-700"><svg class="mr-1.5 text-indigo-500 inline-block -mt-0.5" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width=".8em" height=".8em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 24 24"><path class="uim-quaternary" d="M20.23 7.24L12 12L3.77 7.24a1.98 1.98 0 0 1 .7-.71L11 2.76c.62-.35 1.38-.35 2 0l6.53 3.77c.29.173.531.418.7.71z" opacity=".25" fill="currentColor"></path><path class="uim-tertiary" d="M12 12v9.5a2.09 2.09 0 0 1-.91-.21L4.5 17.48a2.003 2.003 0 0 1-1-1.73v-7.5a2.06 2.06 0 0 1 .27-1.01L12 12z" opacity=".5" fill="currentColor"></path><path class="uim-primary" d="M20.5 8.25v7.5a2.003 2.003 0 0 1-1 1.73l-6.62 3.82c-.275.13-.576.198-.88.2V12l8.23-4.76c.175.308.268.656.27 1.01z" fill="currentColor"></path></svg><span class="font-light">kelas</span> <span class="font-medium">diffusers.</span><span class="font-semibold">Cosmos3OmniModularPipeline</span></span></h3><!----> <a id="diffusers.Cosmos3OmniModularPipeline" class="header-link invisible with-hover:group-hover:visible pr-2" href="#diffusers.Cosmos3OmniModularPipeline"><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></a> <!--[0--><a class="!ml-auto !text-gray-400 !no-underline text-sm flex items-center" href="https://github.com/huggingface/diffusers/blob/main/src/diffusers/modular_pipelines/cosmos/modular_pipeline.py#L7" target="_blank"><span><</span> <span class="hidden md:block mx-0.5 hover:!underline">source</span> <span>></span></a><!--]--></span> <!--[0--><p class="font-mono text-xs md:text-sm !leading-relaxed !my-6"><span>(</span> <!--[--><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">blocks<span class="opacity-60">: diffusers.modular_pipelines.modular_pipeline.ModularPipelineBlocks | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">pretrained_model_name_or_path<span class="opacity-60">: str | os.PathLike | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">components_manager<span class="opacity-60">: diffusers.modular_pipelines.components_manager.ComponentsManager | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">collection<span class="opacity-60">: str | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">workflow<span class="opacity-60">: str | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">modular_config_dict<span class="opacity-60">: dict[str, typing.Any] | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">config_dict<span class="opacity-60">: dict[str, typing.Any] | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">**kwargs<span class="opacity-60"></span></span></span><!--]--> <span>)</span> <!--[-1--><!--]--></p><!--]--> <div class="!mb-10 relative docstring-details "><!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--></div></div><!----> <p>ModularPipeline untuk generasi omni Cosmos 3.</p></div> <ul><li>semua</li> <li><strong>call</strong></li></ul> <!--[1--><h2 class="relative group"><a id="diffusers.Cosmos3DistilledModularPipeline" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#diffusers.Cosmos3DistilledModularPipeline"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Cosmos3DistilledModularPipeline<!----></span></h2><!--]--><!----> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><div><span class="group flex space-x-1.5 items-center text-gray-800 bg-gradient-to-r rounded-tr-lg -mt-4 -ml-4 pt-3 px-2.5" id="diffusers.Cosmos3DistilledModularPipeline"><!----><h3 class="!m-0"><span class="flex-1 break-all md:text-lg bg-gradient-to-r px-2.5 py-1.5 rounded-xl from-indigo-50/70 to-white dark:from-gray-900 dark:to-gray-950 dark:text-indigo-300 text-indigo-700"><svg class="mr-1.5 text-indigo-500 inline-block -mt-0.5" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width=".8em" height=".8em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 24 24"><path class="uim-quaternary" d="M20.23 7.24L12 12L3.77 7.24a1.98 1.98 0 0 1 .7-.71L11 2.76c.62-.35 1.38-.35 2 0l6.53 3.77c.29.173.531.418.7.71z" opacity=".25" fill="currentColor"></path><path class="uim-tertiary" d="M12 12v9.5a2.09 2.09 0 0 1-.91-.21L4.5 17.48a2.003 2.003 0 0 1-1-1.73v-7.5a2.06 2.06 0 0 1 .27-1.01L12 12z" opacity=".5" fill="currentColor"></path><path class="uim-primary" d="M20.5 8.25v7.5a2.003 2.003 0 0 1-1 1.73l-6.62 3.82c-.275.13-.576.198-.88.2V12l8.23-4.76c.175.308.268.656.27 1.01z" fill="currentColor"></path></svg><span class="font-light">kelas</span> <span class="font-medium">diffusers.</span><span class="font-semibold">Cosmos3DistilledModularPipeline</span></span></h3><!----> <a id="diffusers.Cosmos3DistilledModularPipeline" class="header-link invisible with-hover:group-hover:visible pr-2" href="#diffusers.Cosmos3DistilledModularPipeline"><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></a> <!--[0--><a class="!ml-auto !text-gray-400 !no-underline text-sm flex items-center" href="https://github.com/huggingface/diffusers/blob/main/src/diffusers/modular_pipelines/cosmos/modular_pipeline.py#L121" target="_blank"><span><</span> <span class="hidden md:block mx-0.5 hover:!underline">source</span> <span>></span></a><!--]--></span> <!--[0--><p class="font-mono text-xs md:text-sm !leading-relaxed !my-6"><span>(</span> <!--[--><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">blocks<span class="opacity-60">: diffusers.modular_pipelines.modular_pipeline.ModularPipelineBlocks | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">pretrained_model_name_or_path<span class="opacity-60">: str | os.PathLike | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">components_manager<span class="opacity-60">: diffusers.modular_pipelines.components_manager.ComponentsManager | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">collection<span class="opacity-60">: str | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">workflow<span class="opacity-60">: str | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">modular_config_dict<span class="opacity-60">: dict[str, typing.Any] | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">config_dict<span class="opacity-60">: dict[str, typing.Any] | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">**kwargs<span class="opacity-60"></span></span></span><!--]--> <span>)</span> <!--[-1--><!--]--></p><!--]--> <div class="!mb-10 relative docstring-details "><!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--></div></div><!----> <p>ModularPipeline untuk generasi Cosmos 3 omni yang telah di-distilasi (sedikit langkah).</p> <p>Checkpoint yang telah di-distil membakar classifier-free guidance ke dalam bobot dan melakukan sampling pada jadwal tetap yang dibaca dari konfigurasi <code>distilled_sigmas</code> pipeline (diisi dari <code>modular_model_index.json</code>), sehingga <code>guidance_scale</code> dan <code>num_inference_steps</code> bersifat tetap dan <code>negative_prompt</code> tidak didukung.</p></div> <ul><li>semua</li> <li><strong>call</strong></li></ul> <!--[1--><h2 class="relative group"><a id="diffusers.CosmosActionCondition" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#diffusers.CosmosActionCondition"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->CosmosActionCondition<!----></span></h2><!--]--><!----> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><div><span class="group flex space-x-1.5 items-center text-gray-800 bg-gradient-to-r rounded-tr-lg -mt-4 -ml-4 pt-3 px-2.5" id="diffusers.CosmosActionCondition"><!----><h3 class="!m-0"><span class="flex-1 break-all md:text-lg bg-gradient-to-r px-2.5 py-1.5 rounded-xl from-indigo-50/70 to-white dark:from-gray-900 dark:to-gray-950 dark:text-indigo-300 text-indigo-700"><svg class="mr-1.5 text-indigo-500 inline-block -mt-0.5" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width=".8em" height=".8em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 24 24"><path class="uim-quaternary" d="M20.23 7.24L12 12L3.77 7.24a1.98 1.98 0 0 1 .7-.71L11 2.76c.62-.35 1.38-.35 2 0l6.53 3.77c.29.173.531.418.7.71z" opacity=".25" fill="currentColor"></path><path class="uim-tertiary" d="M12 12v9.5a2.09 2.09 0 0 1-.91-.21L4.5 17.48a2.003 2.003 0 0 1-1-1.73v-7.5a2.06 2.06 0 0 1 .27-1.01L12 12z" opacity=".5" fill="currentColor"></path><path class="uim-primary" d="M20.5 8.25v7.5a2.003 2.003 0 0 1-1 1.73l-6.62 3.82c-.275.13-.576.198-.88.2V12l8.23-4.76c.175.308.268.656.27 1.01z" fill="currentColor"></path></svg><span class="font-light">kelas</span> <span class="font-medium">diffusers.</span><span class="font-semibold">CosmosActionCondition</span></span></h3><!----> <a id="diffusers.CosmosActionCondition" class="header-link invisible with-hover:group-hover:visible pr-2" href="#diffusers.CosmosActionCondition"><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></a> <!--[0--><a class="!ml-auto !text-gray-400 !no-underline text-sm flex items-center" href="https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/cosmos/pipeline_cosmos3_omni.py#L302" target="_blank"><span><</span> <span class="hidden md:block mx-0.5 hover:!underline">source</span> <span>></span></a><!--]--></span> <!--[0--><p class="font-mono text-xs md:text-sm !leading-relaxed !my-6"><span>(</span> <!--[--><span class="comma cursor-pointer"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">mode<span class="opacity-60">: typing.Literal['policy', 'forward_dynamics', 'inverse_dynamics']</span></span></span><span class="comma cursor-pointer"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">chunk_size<span class="opacity-60">: int</span></span></span><span class="comma cursor-pointer"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">domain_name<span class="opacity-60">: str</span></span></span><span class="comma cursor-pointer"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">resolution_tier<span class="opacity-60">: int = 480</span></span></span><span class="comma cursor-pointer"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">raw_actions<span class="opacity-60">: typing.Optional[torch.Tensor] = None</span></span></span><span class="comma cursor-pointer"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">image<span class="opacity-60">: typing.Union[PIL.Image.Image, numpy.ndarray, torch.Tensor, NoneType] = None</span></span></span><span class="comma cursor-pointer"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">video<span class="opacity-60">: typing.Union[list, numpy.ndarray, torch.Tensor, NoneType] = None</span></span></span><span class="comma cursor-pointer"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">view_point<span class="opacity-60">: str = 'ego_view'</span></span></span><!--]--> <span>)</span> <!--[-1--><!--]--></p><!--]--> <div class="!mb-10 relative docstring-details "><!--[-1--><!--]--> <!--[0--><p class="flex items-center font-semibold !mt-2 !mb-2 text-gray-800">Parameter <span class="flex-auto border-t-2 border-gray-100 dark:border-gray-700 ml-3"></span></p> <ul class="px-2"><!--[--><li class="text-base !pl-4 my-3 rounded "><span class="group flex space-x-1.5 items-start"><a id="diffusers.CosmosActionCondition.mode" class="header-link block pr-0.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#diffusers.CosmosActionCondition.mode"><span><svg class="text-smd" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!----><strong>mode</strong> (<code>str</code>) — | |
| Tugas aksi. Salah satu dari <code>"forward_dynamics"</code> (menghasilkan video masa depan dari bingkai pertama dan urutan | |
| <code>raw_actions</code> yang diberikan), <code>"inverse_dynamics"</code> (menyimpulkan aksi yang menghubungkan bingkai kondisioning), atau | |
| <code>"policy"</code> (secara bersama menghasilkan video masa depan dan aksi dari bingkai pertama).<!----></span></span></li><li class="text-base !pl-4 my-3 rounded "><span class="group flex space-x-1.5 items-start"><a id="diffusers.CosmosActionCondition.chunk_size" class="header-link block pr-0.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#diffusers.CosmosActionCondition.chunk_size"><span><svg class="text-smd" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!----><strong>chunk_size</strong> (<code>int</code>) — | |
| Jumlah langkah transisi aksi dalam chunk. Video kondisioning berpasangan mencakup <code>chunk_size + 1</code> | |
| frame.<!----></span></span></li><li class="text-base !pl-4 my-3 rounded "><span class="group flex space-x-1.5 items-start"><a id="diffusers.CosmosActionCondition.domain_name" class="header-link block pr-0.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#diffusers.CosmosActionCondition.domain_name"><span><svg class="text-smd" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!----><strong>domain_name</strong> (<code>str</code>) — | |
| Domain embodiment yang memilih bobot proyeksi aksi yang sadar domain. Harus menjadi salah satu domain embodiment Cosmos 3 yang terdaftar. Ini juga menetapkan lebar aksi tanpa padding yang digunakan untuk memotong aksi prediksi, yang dipecahkan secara internal dari nama ini (lihat <code>_EMBODIMENT_TO_RAW_ACTION_DIM</code>).<!----></span></span></li><li class="text-base !pl-4 my-3 rounded "><span class="group flex space-x-1.5 items-start"><a id="diffusers.CosmosActionCondition.resolution_tier" class="header-link block pr-0.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#diffusers.CosmosActionCondition.resolution_tier"><span><svg class="text-smd" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!----><strong>resolution_tier</strong> (<code>int</code>, default <code>480</code>) — | |
| Tingkat resolusi untuk kondisioning aksi <em>tier</em> (salah satu dari <code>256</code>, <code>480</code>, <code>704</code>, <code>720</code>). Tier memilih kanvas predefined yang rasio aspeknya paling mendekati input; input akan di-downscale (tidak pernah di-upscale) dan di-pad ke dalamnya untuk kondisioning. Ini bukan ukuran frame output, yang mengikuti konten input. Sesuaikan tier dengan resolusi asli input: tier yang lebih rendah akan membuang detail, sedangkan tier yang lebih tinggi tidak menambah resolusi (tanpa upscaling), membuang komputasi untuk padding, dan merupakan ketidakcocokan antara pelatihan dan inferensi yang dapat menurunkan kualitas.<!----></span></span></li><li class="text-base !pl-4 my-3 rounded "><span class="group flex space-x-1.5 items-start"><a id="diffusers.CosmosActionCondition.raw_actions" class="header-link block pr-0.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#diffusers.CosmosActionCondition.raw_actions"><span><svg class="text-smd" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!----><strong>raw_actions</strong> (<code>torch.Tensor</code>, <em>opsional</em>) — | |
| Vektor aksi domain mentah dengan bentuk <code>[T, raw_action_dim]</code> yang menggerakkan <code>"forward_dynamics"</code>. Urutan yang lebih pendek | |
| dari <code>chunk_size</code> akan mengulang aksi terakhir; yang lebih panjang akan dipangkas. Kanal di luar | |
| <code>action_dim</code> model akan ditolak, dan input yang lebih sempit akan di-nol-padded hingga <code>action_dim</code>.<!----></span></span></li><li class="text-base !pl-4 my-3 rounded "><span class="group flex space-x-1.5 items-start"><a id="diffusers.CosmosActionCondition.image" class="header-link block pr-0.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#diffusers.CosmosActionCondition.image"><span><svg class="text-smd" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!----><strong>image</strong> (<code>PIL.Image.Image</code>, <code>np.ndarray</code>, atau <code>torch.Tensor</code>, <em>opsional</em>) — | |
| Frame kondisioning untuk <code>"policy"</code> / <code>"forward_dynamics"</code>. Saling eksklusif dengan <code>video</code>.<!----></span></span></li><li class="text-base !pl-4 my-3 rounded "><span class="group flex space-x-1.5 items-start"><a id="diffusers.CosmosActionCondition.video" class="header-link block pr-0.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#diffusers.CosmosActionCondition.video"><span><svg class="text-smd" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!----><strong>video</strong> (<code>list</code>, <code>np.ndarray</code>, atau <code>torch.Tensor</code>, <em>opsional</em>) — | |
| Video kondisioning, diperlukan untuk <code>"inverse_dynamics"</code>. Untuk <code>"policy"</code> / <code>"forward_dynamics"</code>, hanya frame pertamanya yang digunakan. Saling eksklusif dengan <code>image</code>.<!----></span></span></li><li class="text-base !pl-4 my-3 rounded "><span class="group flex space-x-1.5 items-start"><a id="diffusers.CosmosActionCondition.view_point" class="header-link block pr-0.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#diffusers.CosmosActionCondition.view_point"><span><svg class="text-smd" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!----><strong>view_point</strong> (<code>str</code>, defaultnya <code>"ego_view"</code>) — | |
| Label perspektif kamera yang digunakan untuk mengisi field <code>cinematography.framing</code> pada caption aksi. Salah satu dari | |
| <code>"ego_view"</code>, <code>"third_person_view"</code>, <code>"wrist_view"</code>, atau <code>"concat_view"</code>. Model aksi dilatih pada | |
| caption JSON terstruktur yang membawa kalimat sudut pandang ini; label yang tidak dikenali akan menghapus field framing | |
| (dengan peringatan).<!----></span></span></li><!--]--></ul><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--></div></div><!----> <p>Mengelompokkan semua input yang diperlukan untuk tugas generasi ber-kondisi aksi Cosmos 3.</p> <p>Lewatkan ini ke <code>Cosmos3OmniPipeline.__call__()</code> melalui argumen <code>action</code> alih-alih argumen tingkat atas <code>image</code> / <code>height</code> / <code>width</code>, yang dicadangkan untuk eksekusi t2v, i2v.</p></div> <!--[1--><h2 class="relative group"><a id="diffusers.pipelines.cosmos.pipeline_cosmos3_omni.Cosmos3OmniPipelineOutput" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#diffusers.pipelines.cosmos.pipeline_cosmos3_omni.Cosmos3OmniPipelineOutput"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Cosmos3OmniPipelineOutput<!----></span></h2><!--]--><!----> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><div><span class="group flex space-x-1.5 items-center text-gray-800 bg-gradient-to-r rounded-tr-lg -mt-4 -ml-4 pt-3 px-2.5" id="diffusers.pipelines.cosmos.pipeline_cosmos3_omni.Cosmos3OmniPipelineOutput"><!----><h3 class="!m-0"><span class="flex-1 break-all md:text-lg bg-gradient-to-r px-2.5 py-1.5 rounded-xl from-indigo-50/70 to-white dark:from-gray-900 dark:to-gray-950 dark:text-indigo-300 text-indigo-700"><svg class="mr-1.5 text-indigo-500 inline-block -mt-0.5" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width=".8em" height=".8em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 24 24"><path class="uim-quaternary" d="M20.23 7.24L12 12L3.77 7.24a1.98 1.98 0 0 1 .7-.71L11 2.76c.62-.35 1.38-.35 2 0l6.53 3.77c.29.173.531.418.7.71z" opacity=".25" fill="currentColor"></path><path class="uim-tertiary" d="M12 12v9.5a2.09 2.09 0 0 1-.91-.21L4.5 17.48a2.003 2.003 0 0 1-1-1.73v-7.5a2.06 2.06 0 0 1 .27-1.01L12 12z" opacity=".5" fill="currentColor"></path><path class="uim-primary" d="M20.5 8.25v7.5a2.003 2.003 0 0 1-1 1.73l-6.62 3.82c-.275.13-.576.198-.88.2V12l8.23-4.76c.175.308.268.656.27 1.01z" fill="currentColor"></path></svg><span class="font-light">kelas</span> <span class="font-medium">diffusers.pipelines.cosmos.pipeline_cosmos3_omni.</span><span class="font-semibold">Cosmos3OmniPipelineOutput</span></span></h3><!----> <a id="diffusers.pipelines.cosmos.pipeline_cosmos3_omni.Cosmos3OmniPipelineOutput" class="header-link invisible with-hover:group-hover:visible pr-2" href="#diffusers.pipelines.cosmos.pipeline_cosmos3_omni.Cosmos3OmniPipelineOutput"><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></a> <!--[0--><a class="!ml-auto !text-gray-400 !no-underline text-sm flex items-center" href="https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/cosmos/pipeline_cosmos3_omni.py#L283" target="_blank"><span><</span> <span class="hidden md:block mx-0.5 hover:!underline">source</span> <span>></span></a><!--]--></span> <!--[0--><p class="font-mono text-xs md:text-sm !leading-relaxed !my-6"><span>(</span> <!--[--><span class="comma cursor-pointer"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">video<span class="opacity-60">: Any</span></span></span><span class="comma cursor-pointer"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">sound<span class="opacity-60">: typing.Optional[torch.Tensor] = None</span></span></span><span class="comma cursor-pointer"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">action<span class="opacity-60">: list[torch.Tensor] | None = None</span></span></span><!--]--> <span>)</span> <!--[-1--><!--]--></p><!--]--> <div class="!mb-10 relative docstring-details "><!--[-1--><!--]--> <!--[0--><p class="flex items-center font-semibold !mt-2 !mb-2 text-gray-800">Parameter <span class="flex-auto border-t-2 border-gray-100 dark:border-gray-700 ml-3"></span></p> <ul class="px-2"><!--[--><li class="text-base !pl-4 my-3 rounded "><span class="group flex space-x-1.5 items-start"><a id="diffusers.pipelines.cosmos.pipeline_cosmos3_omni.Cosmos3OmniPipelineOutput.video" class="header-link block pr-0.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#diffusers.pipelines.cosmos.pipeline_cosmos3_omni.Cosmos3OmniPipelineOutput.video"><span><svg class="text-smd" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!----><strong>video</strong> — Video yang dihasilkan. Tipe persisnya bergantung pada <code>output_type</code> | |
| yang diteruskan ke pipeline: daftar frame PIL untuk <code>"pil"</code> (default), <code>np.ndarray</code> dengan bentuk <code>[T, H, W, C]</code> untuk <code>"np"</code>, <code>torch.Tensor</code> dengan bentuk <code>[T, C, H, W]</code> untuk <code>"pt"</code>, atau tensor laten mentah | |
| ketika <code>output_type="latent"</code>.<!----></span></span></li><li class="text-base !pl-4 my-3 rounded "><span class="group flex space-x-1.5 items-start"><a id="diffusers.pipelines.cosmos.pipeline_cosmos3_omni.Cosmos3OmniPipelineOutput.sound" class="header-link block pr-0.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#diffusers.pipelines.cosmos.pipeline_cosmos3_omni.Cosmos3OmniPipelineOutput.sound"><span><svg class="text-smd" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!----><strong>sound</strong> — Gelombang audio terdekoding dengan bentuk <code>[C, N]</code>. <code>None</code> ketika | |
| <code>enable_sound=False</code>.<!----></span></span></li><li class="text-base !pl-4 my-3 rounded "><span class="group flex space-x-1.5 items-start"><a id="diffusers.pipelines.cosmos.pipeline_cosmos3_omni.Cosmos3OmniPipelineOutput.action" class="header-link block pr-0.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#diffusers.pipelines.cosmos.pipeline_cosmos3_omni.Cosmos3OmniPipelineOutput.action"><span><svg class="text-smd" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!----><strong>action</strong> — Token aksi yang diprediksi. <code>None</code> kecuali mode aksi memprediksi aksi.<!----></span></span></li><!--]--></ul><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--></div></div><!----> <p>Dataclass output untuk <a href="/docs/diffusers/main/id/api/pipelines/cosmos3#diffusers.Cosmos3OmniPipeline">Cosmos3OmniPipeline</a>.</p></div> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/diffusers/blob/main/docs/source/en/api/pipelines/cosmos3.md" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg><!----> <span><span class="underline">Update</span> on GitHub</span></a><!----> <p></p><!--]--><!----><!--]--><!--]--><!--]--> <!--[-1--><!--]--><!--]--> | |
| <script> | |
| { | |
| __sveltekit_gxjg5b = { | |
| base: "/docs/diffusers/main/en", | |
| assets: "/docs/diffusers/main/en" | |
| }; | |
| const element = document.currentScript.parentElement; | |
| Promise.all([ | |
| import("/docs/diffusers/main/en/_app/immutable/entry/start.U6TUchwJ.js"), | |
| import("/docs/diffusers/main/en/_app/immutable/entry/app.3qKhPoX-.js") | |
| ]).then(([kit, app]) => { | |
| kit.start(app, element, { | |
| node_ids: [0, 7], | |
| data: [null,null], | |
| form: null, | |
| error: null | |
| }); | |
| }); | |
| } | |
| </script> | |
Xet Storage Details
- Size:
- 231 kB
- Xet hash:
- bbdb980c063eee1d6da65de18203b116207cb61661a0c7f06c74a31f574c2e72
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.