Buckets:
| <meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"Cosmos 3","local":"cosmos-3","sections":[{"title":"Nouveautés de Cosmos 3","local":"whats-new-in-cosmos-3","sections":[],"depth":2},{"title":"Points de contrôle disponibles","local":"available-checkpoints","sections":[],"depth":2},{"title":"Dénbruitage mixte FP8 W8A8/W8A16","local":"fp8-mixed-w8a8w8a16-denoising","sections":[],"depth":2},{"title":"Échantillonnage supérieur des prompts","local":"prompt-upsampling","sections":[],"depth":2},{"title":"Texte vers vidéo","local":"text-to-video","sections":[],"depth":2},{"title":"Texte vers image","local":"text-to-image","sections":[],"depth":2},{"title":"Image vers vidéo","local":"image-to-video","sections":[],"depth":2},{"title":"Vidéo-vers-vidéo","local":"video-to-video","sections":[],"depth":2},{"title":"Vidéo-vers-vidéo avec son","local":"video-to-video-with-sound","sections":[],"depth":2},{"title":"Texte vers vidéo avec son","local":"text-to-video-with-sound","sections":[],"depth":2},{"title":"Génération conditionnée par l'action","local":"action-conditioned-generation","sections":[{"title":"Politique d'action","local":"action-policy","sections":[],"depth":3}],"depth":2},{"title":"SeaCache","local":"seacache","sections":[],"depth":2},{"title":"Parallélisme de contexte","local":"context-parallelism","sections":[{"title":"L'exécuter","local":"run-it","sections":[],"depth":3},{"title":"Ajustement de grands modèles avec le parallélisme de tenseurs","local":"fitting-large-models-with-tensor-parallelism","sections":[],"depth":3},{"title":"L'utiliser dans votre propre pipeline modulaire","local":"use-it-in-your-own-modular-pipeline","sections":[],"depth":3}],"depth":2},{"title":"Modèles de métadonnées","local":"metadata-templates","sections":[],"depth":2},{"title":"Vérificateur de sécurité","local":"safety-checker","sections":[],"depth":2},{"title":"Cosmos3OmniPipeline","local":"diffusers.Cosmos3OmniPipeline","sections":[],"depth":2},{"title":"Cosmos3OmniModularPipeline","local":"cosmos3omnimodularpipeline","sections":[{"title":"Exemples modulaires pour tous les flux de travail existants","local":"modular-examples-for-all-existing-workflows","sections":[],"depth":3},{"title":"Transfert modulaire (contrôle structurel)","local":"modular-transfer-structural-control","sections":[],"depth":3},{"title":"Texte vers image et image vers vidéo distillés (peu d'étapes)","local":"diffusers.Cosmos3OmniModularPipeline","sections":[],"depth":3}],"depth":2},{"title":"Cosmos3DistilledModularPipeline","local":"diffusers.Cosmos3DistilledModularPipeline","sections":[],"depth":2},{"title":"CosmosActionCondition","local":"diffusers.CosmosActionCondition","sections":[],"depth":2},{"title":"Cosmos3OmniPipelineOutput","local":"diffusers.pipelines.cosmos.pipeline_cosmos3_omni.Cosmos3OmniPipelineOutput","sections":[],"depth":2}],"depth":1}"/> | |
| <link href="/docs/diffusers/main/en/_app/immutable/entry/start.U6TUchwJ.js" rel="modulepreload"> | |
| <link href="/docs/diffusers/main/en/_app/immutable/chunks/yxSHiGDF.js" rel="modulepreload"> | |
| <link href="/docs/diffusers/main/en/_app/immutable/chunks/CPfOwUO_.js" rel="modulepreload"> | |
| <link href="/docs/diffusers/main/en/_app/immutable/entry/app.3qKhPoX-.js" rel="modulepreload"> | |
| <link href="/docs/diffusers/main/en/_app/immutable/chunks/BYe-f74j.js" rel="modulepreload"> | |
| <link href="/docs/diffusers/main/en/_app/immutable/chunks/DnVIJfC5.js" rel="modulepreload"> | |
| <link href="/docs/diffusers/main/en/_app/immutable/chunks/DsnmJJEf.js" rel="modulepreload"> | |
| <link href="/docs/diffusers/main/en/_app/immutable/chunks/D32hXVzh.js" rel="modulepreload"> | |
| <link href="/docs/diffusers/main/en/_app/immutable/nodes/0.DqpuBF3n.js" rel="modulepreload"> | |
| <link href="/docs/diffusers/main/en/_app/immutable/chunks/Dyp9xKq8.js" rel="modulepreload"> | |
| <link href="/docs/diffusers/main/en/_app/immutable/nodes/7.MVmUqGUj.js" rel="modulepreload"> | |
| <link href="/docs/diffusers/main/en/_app/immutable/chunks/Dk44y1gU.js" rel="modulepreload"> | |
| <link href="/docs/diffusers/main/en/_app/immutable/chunks/CYniLWKs.js" rel="modulepreload"> | |
| <!--if5dit--><meta name="hf:doc:metadata" content="{"title":"Cosmos 3","local":"cosmos-3","sections":[{"title":"What’s new in Cosmos 3","local":"whats-new-in-cosmos-3","sections":[],"depth":2},{"title":"Available checkpoints","local":"available-checkpoints","sections":[],"depth":2},{"title":"FP8 mixed W8A8/W8A16 denoising","local":"fp8-mixed-w8a8w8a16-denoising","sections":[],"depth":2},{"title":"Prompt upsampling","local":"prompt-upsampling","sections":[],"depth":2},{"title":"Text-to-video","local":"text-to-video","sections":[],"depth":2},{"title":"Text-to-image","local":"text-to-image","sections":[],"depth":2},{"title":"Image-to-video","local":"image-to-video","sections":[],"depth":2},{"title":"Video-to-video","local":"video-to-video","sections":[],"depth":2},{"title":"Video-to-video with sound","local":"video-to-video-with-sound","sections":[],"depth":2},{"title":"Text-to-video with sound","local":"text-to-video-with-sound","sections":[],"depth":2},{"title":"Action-conditioned generation","local":"action-conditioned-generation","sections":[{"title":"Action policy","local":"action-policy","sections":[],"depth":3}],"depth":2},{"title":"SeaCache","local":"seacache","sections":[],"depth":2},{"title":"Context parallelism","local":"context-parallelism","sections":[{"title":"Run it","local":"run-it","sections":[],"depth":3},{"title":"Fitting large models with tensor parallelism","local":"fitting-large-models-with-tensor-parallelism","sections":[],"depth":3},{"title":"Use it in your own modular pipeline","local":"use-it-in-your-own-modular-pipeline","sections":[],"depth":3}],"depth":2},{"title":"Metadata templates","local":"metadata-templates","sections":[],"depth":2},{"title":"Safety checker","local":"safety-checker","sections":[],"depth":2},{"title":"Cosmos3OmniPipeline","local":"diffusers.Cosmos3OmniPipeline","sections":[],"depth":2},{"title":"Cosmos3OmniModularPipeline","local":"cosmos3omnimodularpipeline","sections":[{"title":"Modular examples for all existing workflows","local":"modular-examples-for-all-existing-workflows","sections":[],"depth":3},{"title":"Modular transfer (structural control)","local":"modular-transfer-structural-control","sections":[],"depth":3},{"title":"Distilled (few-step) text-to-image and image-to-video","local":"diffusers.Cosmos3OmniModularPipeline","sections":[],"depth":3}],"depth":2},{"title":"Cosmos3DistilledModularPipeline","local":"diffusers.Cosmos3DistilledModularPipeline","sections":[],"depth":2},{"title":"CosmosActionCondition","local":"diffusers.CosmosActionCondition","sections":[],"depth":2},{"title":"Cosmos3OmniPipelineOutput","local":"diffusers.pipelines.cosmos.pipeline_cosmos3_omni.Cosmos3OmniPipelineOutput","sections":[],"depth":2}],"depth":1}"/><!----> | |
| <link href="/docs/diffusers/main/en/_app/immutable/assets/0.tn0RQdqM.css" rel="modulepreload"> <!--[--><!--[0--><!--[--><!--[0--><!--[--><p></p> <!--[0--><h1 class="relative group"><a id="cosmos-3" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#cosmos-3"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Cosmos 3<!----></span></h1><!--]--><!----> <p>NVIDIA Cosmos 3 est un modèle de fondation unifié pour le monde (WFM) destiné à l'IA physique — un modèle omni unique qui combine la génération de mondes, le raisonnement physique et la génération d'actions. Il remplace les modèles distincts Predict, Reason et Transfer des versions précédentes de Cosmos : que vous développiez pour la robotique, les véhicules autonomes ou les espaces intelligents, Cosmos 3 vous offre une base unique pour simuler et comprendre le monde physique.</p> <p>Ce qui est livré avec cette version :</p> <ul><li>Modèles sur le Hugging Face Hub avec fiches de modèle et licences</li> <li>Intégration de Cosmos 3 Diffusers pour les pipelines de génération (cette page)</li> <li>Scripts d'entraînement postérieur pour l'ajustement fin de Cosmos 3 sur vos propres données</li> <li>Jeux de données ouverts de génération de données synthétiques (SDG) pour l'IA physique</li></ul> <!--[1--><h2 class="relative group"><a id="whats-new-in-cosmos-3" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#whats-new-in-cosmos-3"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Nouveautés de Cosmos 3<!----></span></h2><!--]--><!----> <p>La plus grande évolution par rapport aux versions précédentes de Cosmos est que Cosmos 3 est un <em>omni-modèle</em>, construit sur une architecture Mixture-of-Transformers (MoT). Auparavant, les développeurs travaillaient avec des modèles distincts pour la génération de mondes (Predict), la génération contrôlée (Transfer), la compréhension de scènes (Reason) et la génération de politiques d'action. Cosmos 3 unifie l'ensemble de ces fonctionnalités dans un seul modèle qui raisonne et génère à travers les modalités lors d'un unique passage avant.</p> <p>À partir d'un seul modèle, vous pouvez :</p> <ul><li>Générer des mondes vidéo physiquement plausibles à partir de texte, d'images ou d'entrées d'action (génération vidéo image-vers-vidéo, texte-vers-vidéo, génération vidéo conditionnée par l'action).</li> <li>Raisonnez sur des propriétés physiques telles que le mouvement, la causalité et les relations spatiales.</li> <li>Prédisez les séquences futures de vidéos et d’actions à partir de l’état actuel.</li> <li>Transférez des scènes entre différents points de vue et conditions avec un contrôle structurel <em>(bientôt disponible)</em>.</li></ul> <p>Sous le capot, une seule instance de <code>Cosmos3OmniTransformer</code> exécute un modèle de langage de type Qwen en parallèle avec un chemin de génération par diffusion : les jetons de texte circulent dans un flux causal d’« compréhension » tandis que les latents vidéo et audio circulent dans un flux de « génération » à attention bidirectionnelle, le tout étant relié par un RoPE multimodal 3D. Consultez le <a href="https://huggingface.co/papers/2501.03575" rel="nofollow">paper de la plateforme Cosmos World Foundation Model</a> pour le contexte architectural.</p> <!--[1--><h2 class="relative group"><a id="available-checkpoints" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#available-checkpoints"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Points de contrôle disponibles<!----></span></h2><!--]--><!----> <p>Deux points de contrôle sont publiés sur le Hub — <a href="https://huggingface.co/nvidia/Cosmos3-Nano" rel="nofollow"><code>nvidia/Cosmos3-Nano</code></a> (plus petit, plus rapide) et <a href="https://huggingface.co/nvidia/Cosmos3-Super" rel="nofollow"><code>nvidia/Cosmos3-Super</code></a> (plus grand, qualité supérieure). La même classe de pipeline prend en charge le texte-vers-image, le texte-vers-vidéo, l'image-vers-vidéo et (avec un point de contrôle capable de générer du son) le texte+image-vers-vidéo-avec-son — choisissez un dépôt et utilisez l'onglet par modèle dans chaque workflow ci-dessous.</p> <blockquote class="tip"><p>Pensez à consulter le <a href="../../using-diffusers/schedulers">guide des Schedulers</a> pour apprendre à explorer le compromis entre la vitesse et la qualité du scheduler, et à consulter la section <a href="../../using-diffusers/loading#reusing-models-in-multiple-pipelines">réutilisation des composants entre les pipelines</a> pour apprendre à charger efficacement les mêmes composants dans plusieurs pipelines.</p></blockquote> <!--[1--><h2 class="relative group"><a id="fp8-mixed-w8a8w8a16-denoising" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#fp8-mixed-w8a8w8a16-denoising"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Dénbruitage mixte FP8 W8A8/W8A16<!----></span></h2><!--]--><!----> <p>Les points de contrôle FP8 ModelOpt officiels pour Nano, Super et Edge sont publiés sous forme de dépôts Hub distincts : <a href="https://huggingface.co/nvidia/Cosmos3-Nano-FP8" rel="nofollow"><code>nvidia/Cosmos3-Nano-FP8</code></a>, <a href="https://huggingface.co/nvidia/Cosmos3-Super-FP8" rel="nofollow"><code>nvidia/Cosmos3-Super-FP8</code></a> et <a href="https://huggingface.co/nvidia/Cosmos3-Edge-FP8" rel="nofollow"><code>nvidia/Cosmos3-Edge-FP8</code></a>. Chargez <code>main</code> du dépôt FP8. <a href="https://huggingface.co/nvidia/Cosmos3-Super-Image2Video" rel="nofollow"><code>nvidia/Cosmos3-Super-Image2Video</code></a> ne dispose pas de dépôt FP8 distinct ; chargez-le avec <code>revision="fp8"</code>.</p> <p>Tous ces points de contrôle sont quantifiés de la même manière. <strong>W8A8</strong> utilise des poids en 8 bits et des activations en 8 bits (le GEMM ModelOpt restauré). <strong>W8A16</strong> réutilise ces mêmes poids en 8 bits mais saute la quantification des activations : le poids FP8 est déquantifié et une opération linéaire standard est exécutée sur des activations BF16/FP16/FP32.</p> <p>L’exécution de W8A8 à chaque étape peut provoquer des scintillements visibles dans la génération de vidéos multi-étapes. Les points de contrôle vidéo Nano, Super, Super-I2V et Edge FP8 déclarent donc un planning dans <code>transformer/config.json</code> : <strong>W8A16 pour les 3 premières et les 3 dernières étapes</strong>, <strong>W8A8 en milieu de séquence</strong>. Diffusers lit ces compteurs depuis le point de contrôle au lieu de les coder en dur. La précision est choisie une fois par étape du planificateur afin que les appels cond/uncond du guidage sans classifieur correspondent.</p> <p>La génération d’images et les points de contrôle distillés en peu d’étapes ne présentent pas ce scintillement, si bien que Super-T2I et les dépôts distillés 4 étapes FP8 ne déclarent aucun calendrier et restent en W8A8 à chaque étape. Le calendrier est également <strong>exclusif à ModelOpt FP8</strong> : les autres backends de quantification (par exemple TorchAO) conservent leurs forwards natifs.</p> <p>Chargez un dépôt FP8 avec le même chemin de restauration que le <a href="../../quantization/modelopt">guide ModelOpt</a>. Le dépôt contient déjà la configuration de quantification. La précision mixte suit alors automatiquement le point de contrôle :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copié</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniPipeline | |
| pipe = Cosmos3OmniPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Nano-FP8"</span>, | |
| dtype=torch.bfloat16, | |
| device_map=<span class="hljs-string">"cuda"</span>, | |
| ) | |
| result = pipe(prompt=<span class="hljs-string">"..."</span>, num_inference_steps=<span class="hljs-number">35</span>)<!----></pre></div><!----> <p>Deux choix au moment de la génération :</p> <ul><li><strong>Par défaut</strong> (<code>mixed_precision_format=None</code>) : si le point de contrôle déclare <code>diffusion_step_policy</code>, exécutez W8A16 sur les N premières/dernières étapes et W8A8 natif en milieu de parcours. C'est la recette prévue pour la vidéo <strong>video</strong> FP8 multi-étapes (moins de scintillement que le tout-W8A8). Les modèles FP8 distillés 4 étapes et Super-T2I omettent la politique, donc le comportement par défaut est déjà tout W8A8.</li> <li><strong><code>mixed_precision_format="none"</code></strong> : conserve chaque étape en W8A8 natif. Plus rapide, car W8A16 correspond à une déquantification + <code>torch.nn.functional.linear</code> plutôt qu’au GEMM FP8 restauré, mais les vidéos multi-étapes peuvent scintiller. Utilisez ce mode pour comparer le planning ou pour correspondre à une référence entièrement quantifiée.</li></ul> <p>Sur une seule station de travail Blackwell, Cosmos3-Nano-FP8 à 720×1280 / 35 étapes était environ <strong>27 % plus lent</strong> (T2I) et <strong>13 % plus lent</strong> (T2V 49 images) avec l’ordonnancement mixte par défaut que avec <code>"none"</code>. Ces chiffres ne constituent pas une garantie de débit. Passez <code>"fp8"</code> uniquement pour forcer l’ordonnancement first/last-N sur un point de contrôle ModelOpt FP8 qui n’a pas de politique.</p> <!--[1--><h2 class="relative group"><a id="prompt-upsampling" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#prompt-upsampling"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Échantillonnage supérieur des prompts<!----></span></h2><!--]--><!----> <p>Cosmos 3 a été entraîné sur des légendes longues et très descriptives. Pour une qualité optimale, les prompts texte courts doivent être <strong>échantillonnés vers le haut dans une structure JSON spécifique</strong> avant d’être transmis au pipeline. L’échantillonneur se trouve dans le package <a href="https://github.com/NVIDIA/cosmos-framework" rel="nofollow">cosmos-framework</a>.</p> <p>Commencez par un court prompt en texte brut et enregistrez-le dans <code>assets/prompt.txt</code>. Pour l’exemple de génération de vidéo à partir de texte ci-dessous, le prompt original est <em>« A robotic arm is cleaning a plate in a kitchen »</em> :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copié</div><!----></button><!----></div> <pre class="language-bash "><!----><span class="hljs-built_in">mkdir</span> -p assets | |
| <span class="hljs-built_in">echo</span> <span class="hljs-string">"A robotic arm is cleaning a plate in a kitchen"</span> > assets/prompt.txt<!----></pre></div><!----> <p>Installez ensuite le framework et exécutez l'upsampler. L'exemple ci-dessous effectue un upsampling pour la génération de vidéo à partir de texte en utilisant Opus-4.6 :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copié</div><!----></button><!----></div> <pre class="language-bash "><!---->git <span class="hljs-built_in">clone</span> https://github.com/NVIDIA/cosmos-framework.git packages/cosmos-framework | |
| pip install -e packages/cosmos-framework | |
| <span class="hljs-built_in">export</span> PROMPT_UPSAMPLER_ENDPOINT_URL=<span class="hljs-string">"https://api.anthropic.com/v1/"</span> | |
| <span class="hljs-built_in">export</span> PROMPT_UPSAMPLER_MODEL_NAME=<span class="hljs-string">"claude-opus-4-6"</span> | |
| <span class="hljs-built_in">export</span> PROMPT_UPSAMPLER_API_TOKEN=<span class="hljs-string">"<your_token>"</span> | |
| python -m cosmos_framework.inference.prompt_upsampling \ | |
| --input assets/prompt.txt \ | |
| --output assets/example_t2v_prompt.json \ | |
| --mode text2video \ | |
| --endpoint-url <span class="hljs-string">"<span class="hljs-variable">${PROMPT_UPSAMPLER_ENDPOINT_URL}</span>"</span> \ | |
| --model <span class="hljs-string">"<span class="hljs-variable">${PROMPT_UPSAMPLER_MODEL_NAME}</span>"</span> \ | |
| --api-token <span class="hljs-string">"<span class="hljs-variable">${PROMPT_UPSAMPLER_API_TOKEN}</span>"</span> \ | |
| --resolution 720 \ | |
| --aspect-ratio <span class="hljs-string">"16,9"</span><!----></pre></div><!----> <p>Basculez <code>--mode</code> pour correspondre au flux de travail visé (<code>text2image</code>, <code>text2video</code>, <code>image2video</code>). La commande écrit les prompts suréchantillonnés dans le fichier <code>--output</code> sous forme de tableau JSON (un objet par ligne non vide de <code>--input</code>) ; passez plutôt un chemin <code>.jsonl</code> pour obtenir un objet JSON par ligne. Pour <code>image2video</code>, vous devez également fournir l’image de conditionnement via <code>--image-url</code> (une URL ou un chemin local) ou <code>--image-list</code> (une image par prompt).</p> <p>Un prompt positif pré-échantillonné (<code>assets/example_t2v_prompt.json</code>) et un prompt négatif (<code>assets/negative_prompt.json</code>) sont fournis pour votre confort et sont utilisés dans les exemples de génération ci-dessous. Les exemples chargent ces fichiers JSON et les transmettent au pipeline sous forme de chaînes JSON via <code>json.dumps(...)</code>.</p> <!--[1--><h2 class="relative group"><a id="text-to-video" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#text-to-video"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Texte vers vidéo<!----></span></h2><!--]--><!----> <p>Génération multi-images conditionnée uniquement sur le texte. Choisissez <code>num_frames</code> en fonction de la durée cible — la valeur par défaut <code>num_frames=189</code> produit ≈ 7,9 s à 24 FPS. Le prompt et le prompt négatif sont lus depuis les fichiers JSON-échantillonnés décrits dans <a href="#prompt-upsampling">Échantillonnage des prompts</a>.</p> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"><!--[--><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd border-gray-800 bg-black dark:bg-gray-700 text-white">Nano</div><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd text-gray-500 cursor-pointer opacity-90 hover:text-gray-700 dark:hover:text-gray-200 hover:shadow-sm">Super</div><!--]--></div> <div class="language-select"><div><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copié</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniPipeline | |
| <span class="hljs-keyword">from</span> diffusers.schedulers.scheduling_unipc_multistep <span class="hljs-keyword">import</span> UniPCMultistepScheduler | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video | |
| <span class="hljs-comment"># JSON-upsampled positive and negative prompts (see "Prompt upsampling" above).</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_t2v_prompt.json"</span>)) | |
| negative_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/negative_prompt.json"</span>)) | |
| pipe = Cosmos3OmniPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Nano"</span>, dtype=torch.bfloat16, device_map=<span class="hljs-string">"cuda"</span> <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| ) | |
| pipe.scheduler = UniPCMultistepScheduler.from_config( | |
| pipe.scheduler.config, flow_shift=<span class="hljs-number">10.0</span>, use_karras_sigmas=<span class="hljs-literal">False</span> | |
| ) | |
| result = pipe( | |
| prompt=json.dumps(json_prompt), | |
| negative_prompt=json.dumps(negative_prompt), | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| num_inference_steps=<span class="hljs-number">35</span>, | |
| guidance_scale=<span class="hljs-number">6.0</span>, | |
| fps=<span class="hljs-number">24.0</span>, | |
| ) | |
| <span class="hljs-comment"># macro_block_size=1 allows arbitrary frame sizes (Cosmos3 outputs are not always divisible by 16).</span> | |
| export_to_video(result.video, <span class="hljs-string">"cosmos3_t2v.mp4"</span>, fps=<span class="hljs-number">24</span>, macro_block_size=<span class="hljs-number">1</span>)<!----></pre></div><!----></div><!----> <div hidden=""><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copié</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniPipeline | |
| <span class="hljs-keyword">from</span> diffusers.schedulers.scheduling_unipc_multistep <span class="hljs-keyword">import</span> UniPCMultistepScheduler | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video | |
| <span class="hljs-comment"># JSON-upsampled positive and negative prompts (see "Prompt upsampling" above).</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_t2v_prompt.json"</span>)) | |
| negative_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/negative_prompt.json"</span>)) | |
| pipe = Cosmos3OmniPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Super"</span>, dtype=torch.bfloat16, device_map=<span class="hljs-string">"cuda"</span> <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| ) | |
| pipe.scheduler = UniPCMultistepScheduler.from_config( | |
| pipe.scheduler.config, flow_shift=<span class="hljs-number">10.0</span>, use_karras_sigmas=<span class="hljs-literal">False</span> | |
| ) | |
| result = pipe( | |
| prompt=json.dumps(json_prompt), | |
| negative_prompt=json.dumps(negative_prompt), | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| num_inference_steps=<span class="hljs-number">35</span>, | |
| guidance_scale=<span class="hljs-number">6.0</span>, | |
| fps=<span class="hljs-number">24.0</span>, | |
| ) | |
| <span class="hljs-comment"># macro_block_size=1 allows arbitrary frame sizes (Cosmos3 outputs are not always divisible by 16).</span> | |
| export_to_video(result.video, <span class="hljs-string">"cosmos3_t2v.mp4"</span>, fps=<span class="hljs-number">24</span>, macro_block_size=<span class="hljs-number">1</span>)<!----></pre></div><!----></div><!----><!----></div><!----> <!--[1--><h2 class="relative group"><a id="text-to-image" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#text-to-image"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Texte vers image<!----></span></h2><!--]--><!----> <p>Génération d'une image unique. Le modèle est conditionné uniquement sur le prompt texte ; passez <code>num_frames=1</code>. Utilisez <code>--mode text2image</code> pour générer le prompt JSON.</p> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"><!--[--><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd border-gray-800 bg-black dark:bg-gray-700 text-white">Nano</div><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd text-gray-500 cursor-pointer opacity-90 hover:text-gray-700 dark:hover:text-gray-200 hover:shadow-sm">Super</div><!--]--></div> <div class="language-select"><div><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copié</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniPipeline | |
| <span class="hljs-comment"># JSON-upsampled prompt (see "Prompt upsampling" above).</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_t2i_prompt.json"</span>)) | |
| pipe = Cosmos3OmniPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Nano"</span>, dtype=torch.bfloat16, device_map=<span class="hljs-string">"cuda"</span> <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| ) | |
| result = pipe(prompt=json.dumps(json_prompt), num_frames=<span class="hljs-number">1</span>, height=<span class="hljs-number">720</span>, width=<span class="hljs-number">1280</span>) | |
| result.video[<span class="hljs-number">0</span>].save(<span class="hljs-string">"cosmos3_t2i.jpg"</span>, <span class="hljs-built_in">format</span>=<span class="hljs-string">"JPEG"</span>, quality=<span class="hljs-number">85</span>)<!----></pre></div><!----></div><!----> <div hidden=""><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copié</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniPipeline | |
| <span class="hljs-comment"># JSON-upsampled prompt (see "Prompt upsampling" above).</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_t2i_prompt.json"</span>)) | |
| pipe = Cosmos3OmniPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Super"</span>, dtype=torch.bfloat16, device_map=<span class="hljs-string">"cuda"</span> <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| ) | |
| result = pipe(prompt=json.dumps(json_prompt), num_frames=<span class="hljs-number">1</span>, height=<span class="hljs-number">720</span>, width=<span class="hljs-number">1280</span>) | |
| result.video[<span class="hljs-number">0</span>].save(<span class="hljs-string">"cosmos3_t2i.jpg"</span>, <span class="hljs-built_in">format</span>=<span class="hljs-string">"JPEG"</span>, quality=<span class="hljs-number">85</span>)<!----></pre></div><!----></div><!----><!----></div><!----> <!--[1--><h2 class="relative group"><a id="image-to-video" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#image-to-video"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Image vers vidéo<!----></span></h2><!--]--><!----> <p>Passez une image de conditionnement via <code>image=</code>. Le pipeline ancre le cadre 0 à l’image fournie et débruite le reste. L’image est redimensionnée en préservant son ratio d’aspect, recadrée au centre à la taille de sortie demandée, et normalisée avec un arrondi équivalent à uint8. Utilisez <code>--mode image2video</code> pour générer le prompt JSON.</p> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"><!--[--><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd border-gray-800 bg-black dark:bg-gray-700 text-white">Nano</div><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd text-gray-500 cursor-pointer opacity-90 hover:text-gray-700 dark:hover:text-gray-200 hover:shadow-sm">Super</div><!--]--></div> <div class="language-select"><div><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copié</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniPipeline | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video, load_image | |
| <span class="hljs-comment"># JSON-upsampled positive and negative prompts (see "Prompt upsampling" above).</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_i2v_prompt.json"</span>)) | |
| negative_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/negative_prompt_i2v.json"</span>)) | |
| pipe = Cosmos3OmniPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Nano"</span>, dtype=torch.bfloat16, device_map=<span class="hljs-string">"cuda"</span> <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| ) | |
| image = load_image( | |
| <span class="hljs-string">"https://github.com/nvidia-cosmos/cosmos-dependencies/releases/download/assets/robot_153.jpg"</span> | |
| ) | |
| result = pipe( | |
| prompt=json.dumps(json_prompt), | |
| negative_prompt=json.dumps(negative_prompt), | |
| image=image, | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| fps=<span class="hljs-number">24.0</span>, | |
| ) | |
| <span class="hljs-comment"># macro_block_size=1 allows arbitrary frame sizes (Cosmos3 outputs are not always divisible by 16).</span> | |
| export_to_video(result.video, <span class="hljs-string">"cosmos3_i2v.mp4"</span>, fps=<span class="hljs-number">24</span>, macro_block_size=<span class="hljs-number">1</span>)<!----></pre></div><!----></div><!----> <div hidden=""><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copié</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniPipeline | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video, load_image | |
| <span class="hljs-comment"># JSON-upsampled positive and negative prompts (see "Prompt upsampling" above).</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_i2v_prompt.json"</span>)) | |
| negative_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/negative_prompt_i2v.json"</span>)) | |
| pipe = Cosmos3OmniPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Super"</span>, dtype=torch.bfloat16, device_map=<span class="hljs-string">"cuda"</span> <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| ) | |
| image = load_image( | |
| <span class="hljs-string">"https://github.com/nvidia-cosmos/cosmos-dependencies/releases/download/assets/robot_153.jpg"</span> | |
| ) | |
| result = pipe( | |
| prompt=json.dumps(json_prompt), | |
| negative_prompt=json.dumps(negative_prompt), | |
| image=image, | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| fps=<span class="hljs-number">24.0</span>, | |
| ) | |
| <span class="hljs-comment"># macro_block_size=1 allows arbitrary frame sizes (Cosmos3 outputs are not always divisible by 16).</span> | |
| export_to_video(result.video, <span class="hljs-string">"cosmos3_i2v.mp4"</span>, fps=<span class="hljs-number">24</span>, macro_block_size=<span class="hljs-number">1</span>)<!----></pre></div><!----></div><!----><!----></div><!----> <!--[1--><h2 class="relative group"><a id="video-to-video" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#video-to-video"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Vidéo-vers-vidéo<!----></span></h2><!--]--><!----> <p>Passez un clip de conditionnement via <code>video=</code> (par exemple, issu de <code>load_video</code>). Le pipeline ancre les trames latentes initiales définies par <code>condition_frame_indexes_vision</code> (par défaut <code>[0, 1]</code>) au clip et débruite le reste. Utilisez <code>condition_video_keep</code> (<code>"first"</code> ou <code>"last"</code>) pour choisir à quelle extrémité d'un clip source plus long les trames de conditionnement sont prélevées. Comme pour les autres modes, le prompt doit suivre la structure JSON descriptive décrite dans <a href="#prompt-upsampling">Prompt upsampling</a>.</p> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"><!--[--><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd border-gray-800 bg-black dark:bg-gray-700 text-white">Nano</div><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd text-gray-500 cursor-pointer opacity-90 hover:text-gray-700 dark:hover:text-gray-200 hover:shadow-sm">Super</div><!--]--></div> <div class="language-select"><div><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copié</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniPipeline | |
| <span class="hljs-keyword">from</span> diffusers.schedulers.scheduling_unipc_multistep <span class="hljs-keyword">import</span> UniPCMultistepScheduler | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video, load_video | |
| <span class="hljs-comment"># JSON-upsampled positive and negative prompts (see "Prompt upsampling" above).</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_v2v_prompt.json"</span>)) | |
| negative_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/negative_prompt_i2v.json"</span>)) | |
| pipe = Cosmos3OmniPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Nano"</span>, dtype=torch.bfloat16, device_map=<span class="hljs-string">"cuda"</span> <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| ) | |
| pipe.scheduler = UniPCMultistepScheduler.from_config( | |
| pipe.scheduler.config, flow_shift=<span class="hljs-number">10.0</span>, use_karras_sigmas=<span class="hljs-literal">False</span> | |
| ) | |
| video = load_video( | |
| <span class="hljs-string">"https://github.com/nvidia-cosmos/cosmos-dependencies/raw/refs/heads/assets/cosmos3/inputs/vision/robot_pouring.mp4"</span> | |
| ) | |
| result = pipe( | |
| prompt=json.dumps(json_prompt), | |
| negative_prompt=json.dumps(negative_prompt), | |
| video=video, | |
| condition_frame_indexes_vision=[<span class="hljs-number">0</span>, <span class="hljs-number">1</span>], | |
| condition_video_keep=<span class="hljs-string">"first"</span>, | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| num_inference_steps=<span class="hljs-number">35</span>, | |
| guidance_scale=<span class="hljs-number">6.0</span>, | |
| fps=<span class="hljs-number">24.0</span>, | |
| ) | |
| <span class="hljs-comment"># macro_block_size=1 allows arbitrary frame sizes (Cosmos3 outputs are not always divisible by 16).</span> | |
| export_to_video(result.video, <span class="hljs-string">"cosmos3_v2v.mp4"</span>, fps=<span class="hljs-number">24</span>, macro_block_size=<span class="hljs-number">1</span>)<!----></pre></div><!----></div><!----> <div hidden=""><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copié</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniPipeline | |
| <span class="hljs-keyword">from</span> diffusers.schedulers.scheduling_unipc_multistep <span class="hljs-keyword">import</span> UniPCMultistepScheduler | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video, load_video | |
| <span class="hljs-comment"># JSON-upsampled positive and negative prompts (see "Prompt upsampling" above).</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_v2v_prompt.json"</span>)) | |
| negative_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/negative_prompt_i2v.json"</span>)) | |
| pipe = Cosmos3OmniPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Super"</span>, dtype=torch.bfloat16, device_map=<span class="hljs-string">"cuda"</span> <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| ) | |
| pipe.scheduler = UniPCMultistepScheduler.from_config( | |
| pipe.scheduler.config, flow_shift=<span class="hljs-number">10.0</span>, use_karras_sigmas=<span class="hljs-literal">False</span> | |
| ) | |
| video = load_video( | |
| <span class="hljs-string">"https://github.com/nvidia-cosmos/cosmos-dependencies/raw/refs/heads/assets/cosmos3/inputs/vision/robot_pouring.mp4"</span> | |
| ) | |
| result = pipe( | |
| prompt=json.dumps(json_prompt), | |
| negative_prompt=json.dumps(negative_prompt), | |
| video=video, | |
| condition_frame_indexes_vision=[<span class="hljs-number">0</span>, <span class="hljs-number">1</span>], | |
| condition_video_keep=<span class="hljs-string">"first"</span>, | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| num_inference_steps=<span class="hljs-number">35</span>, | |
| guidance_scale=<span class="hljs-number">6.0</span>, | |
| fps=<span class="hljs-number">24.0</span>, | |
| ) | |
| <span class="hljs-comment"># macro_block_size=1 allows arbitrary frame sizes (Cosmos3 outputs are not always divisible by 16).</span> | |
| export_to_video(result.video, <span class="hljs-string">"cosmos3_v2v.mp4"</span>, fps=<span class="hljs-number">24</span>, macro_block_size=<span class="hljs-number">1</span>)<!----></pre></div><!----></div><!----><!----></div><!----> <!--[1--><h2 class="relative group"><a id="video-to-video-with-sound" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#video-to-video-with-sound"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Vidéo-vers-vidéo avec son<!----></span></h2><!--]--><!----> <p>Lorsque le point de contrôle contient un <code>sound_tokenizer</code>, ajoutez <code>enable_sound=True</code> à l’appel vidéo-versus-vidéo pour générer conjointement une piste audio synchronisée. La forme d’onde est renvoyée avec la vidéo et peut être multiplexée dans le MP4 à l’aide de <a href="/docs/diffusers/main/fr/api/utilities#diffusers.utils.encode_video">encode_video()</a>.</p> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"><!--[--><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd border-gray-800 bg-black dark:bg-gray-700 text-white">Nano</div><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd text-gray-500 cursor-pointer opacity-90 hover:text-gray-700 dark:hover:text-gray-200 hover:shadow-sm">Super</div><!--]--></div> <div class="language-select"><div><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copié</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniPipeline | |
| <span class="hljs-keyword">from</span> diffusers.schedulers.scheduling_unipc_multistep <span class="hljs-keyword">import</span> UniPCMultistepScheduler | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> encode_video, load_video | |
| <span class="hljs-comment"># JSON-upsampled positive and negative prompts (see "Prompt upsampling" above).</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_v2v_prompt.json"</span>)) | |
| negative_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/negative_prompt_i2v.json"</span>)) | |
| pipe = Cosmos3OmniPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Nano"</span>, dtype=torch.bfloat16, device_map=<span class="hljs-string">"cuda"</span> <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| ) | |
| pipe.scheduler = UniPCMultistepScheduler.from_config( | |
| pipe.scheduler.config, flow_shift=<span class="hljs-number">10.0</span>, use_karras_sigmas=<span class="hljs-literal">False</span> | |
| ) | |
| video = load_video( | |
| <span class="hljs-string">"https://github.com/nvidia-cosmos/cosmos-dependencies/raw/refs/heads/assets/cosmos3/inputs/vision/robot_pouring.mp4"</span> | |
| ) | |
| result = pipe( | |
| prompt=json.dumps(json_prompt), | |
| negative_prompt=json.dumps(negative_prompt), | |
| video=video, | |
| condition_frame_indexes_vision=[<span class="hljs-number">0</span>, <span class="hljs-number">1</span>], | |
| condition_video_keep=<span class="hljs-string">"first"</span>, | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| fps=<span class="hljs-number">24.0</span>, | |
| enable_sound=<span class="hljs-literal">True</span>, | |
| ) | |
| encode_video( | |
| result.video, | |
| fps=<span class="hljs-number">24</span>, | |
| audio=result.sound, | |
| audio_sample_rate=pipe.sound_tokenizer.config.sampling_rate, | |
| output_path=<span class="hljs-string">"cosmos3_v2v_with_sound.mp4"</span>, | |
| )<!----></pre></div><!----></div><!----> <div hidden=""><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copié</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniPipeline | |
| <span class="hljs-keyword">from</span> diffusers.schedulers.scheduling_unipc_multistep <span class="hljs-keyword">import</span> UniPCMultistepScheduler | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> encode_video, load_video | |
| <span class="hljs-comment"># JSON-upsampled positive and negative prompts (see "Prompt upsampling" above).</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_v2v_prompt.json"</span>)) | |
| negative_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/negative_prompt_i2v.json"</span>)) | |
| pipe = Cosmos3OmniPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Super"</span>, dtype=torch.bfloat16, device_map=<span class="hljs-string">"cuda"</span> <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| ) | |
| pipe.scheduler = UniPCMultistepScheduler.from_config( | |
| pipe.scheduler.config, flow_shift=<span class="hljs-number">10.0</span>, use_karras_sigmas=<span class="hljs-literal">False</span> | |
| ) | |
| video = load_video( | |
| <span class="hljs-string">"https://github.com/nvidia-cosmos/cosmos-dependencies/raw/refs/heads/assets/cosmos3/inputs/vision/robot_pouring.mp4"</span> | |
| ) | |
| result = pipe( | |
| prompt=json.dumps(json_prompt), | |
| negative_prompt=json.dumps(negative_prompt), | |
| video=video, | |
| condition_frame_indexes_vision=[<span class="hljs-number">0</span>, <span class="hljs-number">1</span>], | |
| condition_video_keep=<span class="hljs-string">"first"</span>, | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| fps=<span class="hljs-number">24.0</span>, | |
| enable_sound=<span class="hljs-literal">True</span>, | |
| ) | |
| encode_video( | |
| result.video, | |
| fps=<span class="hljs-number">24</span>, | |
| audio=result.sound, | |
| audio_sample_rate=pipe.sound_tokenizer.config.sampling_rate, | |
| output_path=<span class="hljs-string">"cosmos3_v2v_with_sound.mp4"</span>, | |
| )<!----></pre></div><!----></div><!----><!----></div><!----> <!--[1--><h2 class="relative group"><a id="text-to-video-with-sound" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#text-to-video-with-sound"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Texte vers vidéo avec son<!----></span></h2><!--]--><!----> <p>Lorsque le point de contrôle contient un <code>sound_tokenizer</code>, passez <code>enable_sound=True</code> pour générer conjointement une piste audio synchronisée. La forme d’onde est renvoyée avec la vidéo et peut être multiplexée dans le MP4 à l’aide de <a href="/docs/diffusers/main/fr/api/utilities#diffusers.utils.encode_video">encode_video()</a>.</p> <p>C'est le même appel que l'exemple de texte vers vidéo ci-dessus, avec l'ajout de <code>enable_sound=True</code> :</p> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"><!--[--><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd border-gray-800 bg-black dark:bg-gray-700 text-white">Nano</div><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd text-gray-500 cursor-pointer opacity-90 hover:text-gray-700 dark:hover:text-gray-200 hover:shadow-sm">Super</div><!--]--></div> <div class="language-select"><div><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copié</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniPipeline | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> encode_video | |
| <span class="hljs-comment"># JSON-upsampled positive and negative prompts (see "Prompt upsampling" above).</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_t2v_sound_prompt.json"</span>)) | |
| negative_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/negative_prompt.json"</span>)) | |
| pipe = Cosmos3OmniPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Nano"</span>, dtype=torch.bfloat16, device_map=<span class="hljs-string">"cuda"</span> <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| ) | |
| result = pipe( | |
| prompt=json.dumps(json_prompt), | |
| negative_prompt=json.dumps(negative_prompt), | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| fps=<span class="hljs-number">24.0</span>, | |
| enable_sound=<span class="hljs-literal">True</span>, | |
| ) | |
| encode_video( | |
| result.video, | |
| fps=<span class="hljs-number">24</span>, | |
| audio=result.sound, | |
| audio_sample_rate=pipe.sound_tokenizer.config.sampling_rate, | |
| output_path=<span class="hljs-string">"cosmos3_with_sound.mp4"</span>, | |
| )<!----></pre></div><!----></div><!----> <div hidden=""><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copié</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniPipeline | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> encode_video | |
| <span class="hljs-comment"># JSON-upsampled positive and negative prompts (see "Prompt upsampling" above).</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_t2v_sound_prompt.json"</span>)) | |
| negative_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/negative_prompt.json"</span>)) | |
| pipe = Cosmos3OmniPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Super"</span>, dtype=torch.bfloat16, device_map=<span class="hljs-string">"cuda"</span> <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| ) | |
| result = pipe( | |
| prompt=json.dumps(json_prompt), | |
| negative_prompt=json.dumps(negative_prompt), | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| fps=<span class="hljs-number">24.0</span>, | |
| enable_sound=<span class="hljs-literal">True</span>, | |
| ) | |
| encode_video( | |
| result.video, | |
| fps=<span class="hljs-number">24</span>, | |
| audio=result.sound, | |
| audio_sample_rate=pipe.sound_tokenizer.config.sampling_rate, | |
| output_path=<span class="hljs-string">"cosmos3_with_sound.mp4"</span>, | |
| )<!----></pre></div><!----></div><!----><!----></div><!----> <!--[1--><h2 class="relative group"><a id="action-conditioned-generation" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#action-conditioned-generation"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Génération conditionnée par l'action<!----></span></h2><!--]--><!----> <p>Action runs group every action-specific input into a <a href="/docs/diffusers/main/fr/api/pipelines/cosmos3#diffusers.CosmosActionCondition">CosmosActionCondition</a> passed via the <code>action</code> argument instead of the top-level <code>image</code> / <code>video</code> / <code>height</code> / <code>width</code> arguments. Set <code>resolution_tier</code> (<code>256</code>/<code>480</code>/<code>704</code>/<code>720</code>) close to the input video’s native resolution; it selects the conditioning canvas. Cosmos 3 supports three action modes — <code>policy</code>, <code>forward_dynamics</code>, and <code>inverse_dynamics</code>. <code>policy</code> and <code>forward_dynamics</code> condition only on the first frame (so an <code>image</code> or a <code>video</code> both work), while <code>inverse_dynamics</code> requires a <code>video</code>. The conditioning video for an action run is set on <code>action.video</code> (or <code>action.image</code>), not on the pipeline’s top-level <code>video</code> argument.</p> <p>Transmettez une simple description de tâche en tant que <code>prompt</code> et sélectionnez la caméra avec <code>action.view_point</code> (par défaut <code>"ego_view"</code> ; également <code>"third_person_view"</code>, <code>"wrist_view"</code>, <code>"concat_view"</code>). Le pipeline transforme ces éléments en la légende JSON structurée sur laquelle le modèle a été entraîné, les prompts d’action ne doivent donc pas être enrichis par un LLM.</p> <!--[2--><h3 class="relative group"><a id="action-policy" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#action-policy"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Politique d'action<!----></span></h3><!--]--><!----> <p>La génération de politique d’action prédit les futurs jetons vidéo et d’action à partir de la première image d’observation, du prompt texte et des métadonnées du domaine d’action. L’exemple ci-dessous utilise le domaine du robot Bridge et écrit le bloc d’actions prédit au format JSON dans l’espace d’actions normalisé par le modèle.</p> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"><!--[--><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd border-gray-800 bg-black dark:bg-gray-700 text-white">Nano</div><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd text-gray-500 cursor-pointer opacity-90 hover:text-gray-700 dark:hover:text-gray-200 hover:shadow-sm">Super</div><!--]--></div> <div class="language-select"><div><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copié</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniPipeline, CosmosActionCondition | |
| <span class="hljs-keyword">from</span> diffusers.schedulers.scheduling_unipc_multistep <span class="hljs-keyword">import</span> UniPCMultistepScheduler | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video, load_video | |
| pipe = Cosmos3OmniPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Nano"</span>, dtype=torch.bfloat16, device_map=<span class="hljs-string">"cuda"</span> <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| ) | |
| pipe.scheduler = UniPCMultistepScheduler.from_config( | |
| pipe.scheduler.config, flow_shift=<span class="hljs-number">10.0</span>, use_karras_sigmas=<span class="hljs-literal">False</span> | |
| ) | |
| prompt = <span class="hljs-string">"Put the pot to the left of the purple item."</span> | |
| video = load_video( | |
| <span class="hljs-string">"https://github.com/nvidia-cosmos/cosmos-dependencies/raw/refs/heads/assets/cosmos3/inputs/action/bridge_20260501_0.mp4"</span> | |
| ) | |
| result = pipe( | |
| prompt=prompt, | |
| action=CosmosActionCondition( | |
| mode=<span class="hljs-string">"policy"</span>, | |
| chunk_size=<span class="hljs-number">16</span>, | |
| domain_name=<span class="hljs-string">"bridge_orig_lerobot"</span>, | |
| resolution_tier=<span class="hljs-number">480</span>, | |
| video=video, | |
| view_point=<span class="hljs-string">"ego_view"</span>, | |
| ), | |
| fps=<span class="hljs-number">5</span>, | |
| num_inference_steps=<span class="hljs-number">30</span>, | |
| guidance_scale=<span class="hljs-number">1.0</span>, | |
| use_system_prompt=<span class="hljs-literal">False</span>, | |
| ) | |
| <span class="hljs-comment"># macro_block_size=1 allows arbitrary frame sizes (Cosmos3 outputs are not always divisible by 16).</span> | |
| export_to_video(result.video, <span class="hljs-string">"sample.mp4"</span>, fps=<span class="hljs-number">5</span>, macro_block_size=<span class="hljs-number">1</span>) | |
| <span class="hljs-keyword">if</span> result.action <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> <span class="hljs-literal">None</span>: | |
| <span class="hljs-keyword">with</span> <span class="hljs-built_in">open</span>(<span class="hljs-string">"sample_action.json"</span>, <span class="hljs-string">"w"</span>) <span class="hljs-keyword">as</span> f: | |
| json.dump(result.action[<span class="hljs-number">0</span>].tolist(), f)<!----></pre></div><!----></div><!----> <div hidden=""><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copié</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniPipeline, CosmosActionCondition | |
| <span class="hljs-keyword">from</span> diffusers.schedulers.scheduling_unipc_multistep <span class="hljs-keyword">import</span> UniPCMultistepScheduler | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video, load_video | |
| pipe = Cosmos3OmniPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Super"</span>, dtype=torch.bfloat16, device_map=<span class="hljs-string">"cuda"</span> <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| ) | |
| pipe.scheduler = UniPCMultistepScheduler.from_config( | |
| pipe.scheduler.config, flow_shift=<span class="hljs-number">10.0</span>, use_karras_sigmas=<span class="hljs-literal">False</span> | |
| ) | |
| prompt = <span class="hljs-string">"Put the pot to the left of the purple item."</span> | |
| video = load_video( | |
| <span class="hljs-string">"https://github.com/nvidia-cosmos/cosmos-dependencies/raw/refs/heads/assets/cosmos3/inputs/action/bridge_20260501_0.mp4"</span> | |
| ) | |
| result = pipe( | |
| prompt=prompt, | |
| action=CosmosActionCondition( | |
| mode=<span class="hljs-string">"policy"</span>, | |
| chunk_size=<span class="hljs-number">16</span>, | |
| domain_name=<span class="hljs-string">"bridge_orig_lerobot"</span>, | |
| resolution_tier=<span class="hljs-number">480</span>, | |
| video=video, | |
| view_point=<span class="hljs-string">"ego_view"</span>, | |
| ), | |
| fps=<span class="hljs-number">5</span>, | |
| num_inference_steps=<span class="hljs-number">30</span>, | |
| guidance_scale=<span class="hljs-number">1.0</span>, | |
| use_system_prompt=<span class="hljs-literal">False</span>, | |
| ) | |
| <span class="hljs-comment"># macro_block_size=1 allows arbitrary frame sizes (Cosmos3 outputs are not always divisible by 16).</span> | |
| export_to_video(result.video, <span class="hljs-string">"sample.mp4"</span>, fps=<span class="hljs-number">5</span>, macro_block_size=<span class="hljs-number">1</span>) | |
| <span class="hljs-keyword">if</span> result.action <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> <span class="hljs-literal">None</span>: | |
| <span class="hljs-keyword">with</span> <span class="hljs-built_in">open</span>(<span class="hljs-string">"sample_action.json"</span>, <span class="hljs-string">"w"</span>) <span class="hljs-keyword">as</span> f: | |
| json.dump(result.action[<span class="hljs-number">0</span>].tolist(), f)<!----></pre></div><!----></div><!----><!----></div><!----> <!--[1--><h2 class="relative group"><a id="seacache" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#seacache"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->SeaCache<!----></span></h2><!--]--><!----> <p>SeaCache est désactivé par défaut. Cosmos 3 prend en charge son activation explicite via <a href="/docs/diffusers/main/fr/api/cache#diffusers.SeaCacheConfig">SeaCacheConfig</a>. SeaCache réutilise les résidus du transformer lorsque l'indicateur Spectral-Evolution-Aware évolue lentement, réduisant ainsi le nombre d'exécutions complètes du transformer. Il calcule cet indicateur à partir des latents visuels bruts, y compris les images de conditionnement propres pour la génération d'images en vidéo. Activez-le sur le transformer à l'aide des callbacks de métadonnées du planificateur fournis par le pipeline :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copié</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniPipeline, SeaCacheConfig | |
| pipe = Cosmos3OmniPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Nano"</span>, dtype=torch.bfloat16, device_map=<span class="hljs-string">"cuda"</span> | |
| ) | |
| pipe.transformer.enable_cache(SeaCacheConfig(threshold=<span class="hljs-number">0.2</span>, max_consecutive_cached=<span class="hljs-number">2</span>))<!----></pre></div><!----> <p>La même API au niveau du modèle fonctionne avec <a href="/docs/diffusers/main/fr/api/pipelines/cosmos3#diffusers.Cosmos3OmniPipeline">Cosmos3OmniPipeline</a>, <a href="/docs/diffusers/main/fr/api/pipelines/cosmos3#diffusers.Cosmos3OmniModularPipeline">Cosmos3OmniModularPipeline</a> et <a href="/docs/diffusers/main/fr/api/pipelines/cosmos3#diffusers.Cosmos3DistilledModularPipeline">Cosmos3DistilledModularPipeline</a>. SeaCache est approximatif et peut modifier les sorties générées. Désactivez-le avec <code>pipe.transformer.disable_cache()</code> lorsque vous avez besoin que chaque étape de débruitage exécute le transformeur complet. L'état du cache est réinitialisé après chaque appel au pipeline, et les branches de guidage conditionnel et non conditionnel conservent des historiques indépendants.</p> <p>Cosmos 3 conserve la porte SeaCache en dehors de ses couches décodeur répétées, ce qui le rend compatible avec la compilation régionale. | |
| Compilez les couches après avoir activé le cache :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copié</div><!----></button><!----></div> <pre class="language-python "><!---->pipe.transformer.compile_repeated_blocks(fullgraph=<span class="hljs-literal">True</span>)<!----></pre></div><!----> <p>SeaCache prend également en charge les utilitaires de parallélisme de contexte Ulysses et de parallélisme de tenseurs basés sur DTensor pour Cosmos 3, documentés ci-dessous. Les décisions de cache sont synchronisées entre les rangs. La compilation du modèle complet et les autres stratégies de fractionnement de modèle ne sont pas revendiquées.</p> <!--[1--><h2 class="relative group"><a id="context-parallelism" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#context-parallelism"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Parallélisme de contexte<!----></span></h2><!--]--><!----> <p>Pour les vidéos longues ou les hautes résolutions, un seul passage avant peut dépasser le budget mémoire et latence d'un GPU. Cosmos 3 prend en charge le <strong>parallélisme de contexte (CP)</strong> pour répartir la dimension de séquence sur plusieurs GPU, en divisant le calcul d'attention afin que chaque appareil ne conserve qu'une portion des jetons.</p> <p>Cosmos 3 prend en charge le parallélisme de contexte <strong>Ulysses</strong> (échange de séquences/têtes all-to-all). L'attention en anneau n'est pas prise en charge.</p> <p>Contrairement à la plupart des modèles diffusers, Cosmos 3 ne <strong>branche pas</strong> le CP dans le transformeur ni dans le chemin déclaratif <code>enable_parallelism()</code> : son attention à requêtes groupées, ses flux de compréhension/génération distincts (le flux de génération fait attention aux deux) et ses longueurs irrégulières par flux ne peuvent pas être exprimés sous forme de <code>_cp_plan</code>. À la place, le modèle expose de petites coutures de shard/gather sans effet, et l’implémentation se trouve dans <a href="https://github.com/huggingface/diffusers/blob/main/examples/cosmos3/cosmos_parallel.py" rel="nofollow"><code>examples/cosmos3/cosmos_parallel.py</code></a> — un module autonome que vous pouvez lire de bout en bout et adapter. Il propose deux axes de sharding orthogonaux et composables :</p> <table><thead><tr><th>Assistant</th><th>Shards</th><th>Utiliser pour</th></tr></thead><tbody><tr><td><code>enable_cosmos3_context_parallel(transformer, cp_mesh)</code></td><td>séquence (CP / Ulysses)</td><td>latence sur un modèle qui tient sur une seule GPU (<code>Nano</code>)</td></tr><tr><td><code>enable_cosmos3_tensor_parallel(transformer, tp_mesh)</code></td><td>poids (TP)</td><td>entraîner un modèle qui ne tient pas sur une seule GPU (<code>Super</code>)</td></tr></tbody></table> <p>Utilisez l’un ou l’autre seul, ou les deux ensemble, sur un maillage 2-D <code>(tp, cp)</code> (voir <a href="#fitting-large-models-with-tensor-parallelism">Adapter de grands modèles avec le parallélisme de tenseurs</a>).</p> <p>Deux exigences sont spécifiques à Cosmos 3 :</p> <ul><li>Utilisez le backend d’attention <code>native</code>. Cosmos 3 utilise l’attention à requêtes groupées (GQA), et le backend SDPA natif est le seul à accepter <code>enable_gqa</code> (cuDNN et flash le rejettent). Les fonctions utilitaires étendent les têtes KV au nombre de têtes de requête et appellent SDPA avec <code>enable_gqa=False</code> afin qu’il se distribue toujours vers le noyau flash (le repli mathématique matérialiserait les scores complets <code>[S, S]</code> et provoquerait une erreur de mémoire sur les séquences longues).</li> <li>Le degré de CP (Ulysses) doit diviser le nombre de têtes de requête (32 pour <code>Nano</code>, 64 pour <code>Super</code>) ; pour le TP, le degré doit diviser les têtes KV (8). Les flux de compréhension (texte) et de génération (vidéo/son) sont fragmentés indépendamment le long de la séquence, et les longueurs irrégulières sont complétées par des zéros en interne jusqu'à un multiple de la taille du monde.</li></ul> <!--[2--><h3 class="relative group"><a id="run-it" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#run-it"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->L'exécuter<!----></span></h3><!--]--><!----> <p>Le CLI complet <a href="https://github.com/huggingface/diffusers/blob/main/examples/cosmos3/inference_cosmos3.py" rel="nofollow"><code>examples/cosmos3/inference_cosmos3.py</code></a> utilise <a href="/docs/diffusers/main/fr/api/pipelines/cosmos3#diffusers.Cosmos3OmniModularPipeline">Cosmos3OmniModularPipeline</a> et réutilise ces utilitaires, de sorte que <strong>toute modalité</strong> (texte-vers-image/vidéo, image-vers-vidéo, son, modes d'action) s'exécute sur plusieurs GPU via <code>--tp-degree</code> / <code>--cp-degree</code>. Lancez avec <a href="https://docs.pytorch.org/docs/stable/elastic/run.html" rel="nofollow">torchrun</a> ; <code>--tp-degree * --cp-degree</code> doit être égal à <code>--nproc_per_node</code>. Chaque rang produit la même sortie ; le rang 0 l'écrit.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copié</div><!----></button><!----></div> <pre class="language-bash "><!----><span class="hljs-comment"># CP only — Nano (fits one GPU); CP degree must divide 32 query heads.</span> | |
| torchrun --nproc_per_node=4 examples/cosmos3/inference_cosmos3.py --model nano --cp-degree 4 --prompt <span class="hljs-string">"..."</span> | |
| <span class="hljs-comment"># TP only — Super; TP degree must divide 64 query heads and 8 KV heads.</span> | |
| torchrun --nproc_per_node=4 examples/cosmos3/inference_cosmos3.py --model super --tp-degree 4 --prompt <span class="hljs-string">"..."</span> | |
| <span class="hljs-comment"># TP + CP — Super, with sound (TP=2 x CP=2 across 4 GPUs).</span> | |
| torchrun --nproc_per_node=4 examples/cosmos3/inference_cosmos3.py \ | |
| --model super --tp-degree 2 --cp-degree 2 --enable-sound --prompt <span class="hljs-string">"..."</span><!----></pre></div><!----> <p>Les ~120 Go de poids de <code>Super</code> ne tiennent pas sur un seul GPU de 96 Go, il faut donc du TP ; <code>Nano</code> tient sur un GPU unique, donc le CP pour lui est une pure optimisation de latence. (Omettez les deux drapeaux pour exécuter en mono-GPU.)</p> <!--[2--><h3 class="relative group"><a id="fitting-large-models-with-tensor-parallelism" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#fitting-large-models-with-tensor-parallelism"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Ajustement de grands modèles avec le parallélisme de tenseurs<!----></span></h3><!--]--><!----> <p>CP fragmente les <em>activations</em> mais réplique chaque poids sur chaque rang, ce qui ne réduit pas l’empreinte mémoire des poids d’un modèle — un modèle qui ne tient pas sur une seule GPU ne tiendra toujours pas avec CP seul. Pour fragmenter les <strong>poids</strong>, <code>enable_cosmos3_tensor_parallel(transformer, tp_mesh)</code> applique le parallélisme de tenseurs de type Megatron sur un second axe de maillage orthogonal :</p> <ul><li>Les projections d’attention et de MLP sont shardées en colonnes/lignes au sein du groupe TP (<code>to_q/to_k/to_v</code> + <code>add_q/k/v</code> et les <code>gate/up</code> des MLP sont parallèles en colonnes ; <code>to_out/to_add_out</code> et les <code>down</code> des MLP sont parallèles en lignes avec un all-reduce). Chaque rank possède au final <code>query_heads / tp</code> têtes de requête et <code>kv_heads / tp</code> têtes KV.</li> <li>TP se combine avec CP sur un maillage d’appareils 2D <code>(tp, cp)</code> : TP divise de manière persistante les têtes/poids, tandis que CP fragmente la séquence par-dessus. Les contraintes sont que <code>tp</code> divise les têtes KV (8) et que <code>tp * cp</code> divise les têtes de requête (32 pour <code>Nano</code>, 64 pour <code>Super</code>).</li> <li>Les poids sont chargés sur le CPU et répartis sur les GPU couche par couche, de sorte que le modèle complet n'est jamais matérialisé sur un seul appareil.</li></ul> <blockquote class="tip"><p>TP émet un all-reduce sur chaque bloc d’attention et de MLP, ce qui est très gourmand en bande passante. Sur les hôtes sans NVLink, c’est le coût dominant ; préférez le degré de TP le plus petit qui permette aux poids de tenir en mémoire et placez les GPU restants dans CP.</p></blockquote> <!--[2--><h3 class="relative group"><a id="use-it-in-your-own-modular-pipeline" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#use-it-in-your-own-modular-pipeline"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->L'utiliser dans votre propre pipeline modulaire<!----></span></h3><!--]--><!----> <p>Les drapeaux CLI sont pratiques, mais vous pouvez appeler les helpers directement avec <a href="/docs/diffusers/main/fr/api/pipelines/cosmos3#diffusers.Cosmos3OmniModularPipeline">Cosmos3OmniModularPipeline</a>. Chargez la configuration du pipeline et les composants sur le CPU, appliquez le TP <em>avant</em> de déplacer le pipeline sur le GPU local au rang, passez au backend <code>native</code>, puis activez le CP. N'utilisez pas <code>device_map</code> pour ce flux :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copié</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> os | |
| <span class="hljs-keyword">import</span> sys | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">import</span> torch.distributed <span class="hljs-keyword">as</span> dist | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniModularPipeline | |
| <span class="hljs-keyword">from</span> diffusers.schedulers.scheduling_unipc_multistep <span class="hljs-keyword">import</span> UniPCMultistepScheduler | |
| <span class="hljs-keyword">from</span> torch.distributed.device_mesh <span class="hljs-keyword">import</span> init_device_mesh | |
| <span class="hljs-comment"># Make the helper module importable.</span> | |
| sys.path.insert(<span class="hljs-number">0</span>, <span class="hljs-string">"examples/cosmos3"</span>) | |
| <span class="hljs-keyword">from</span> cosmos_parallel <span class="hljs-keyword">import</span> ( | |
| enable_cosmos3_context_parallel, | |
| enable_cosmos3_flash_attention, | |
| enable_cosmos3_tensor_parallel, | |
| ) | |
| <span class="hljs-comment"># torchrun sets RANK / WORLD_SIZE / LOCAL_RANK. Pick tp_degree * cp_degree == world size.</span> | |
| local_rank = <span class="hljs-built_in">int</span>(os.environ[<span class="hljs-string">"LOCAL_RANK"</span>]) | |
| torch.cuda.set_device(local_rank) | |
| dist.init_process_group(<span class="hljs-string">"nccl"</span>) | |
| mesh = init_device_mesh(<span class="hljs-string">"cuda"</span>, (tp_degree, cp_degree), mesh_dim_names=(<span class="hljs-string">"tp"</span>, <span class="hljs-string">"cp"</span>)) | |
| <span class="hljs-comment"># Load components on CPU first; a TP-sharded model may not fit one GPU.</span> | |
| pipe = Cosmos3OmniModularPipeline.from_pretrained(model_id) | |
| pipe.load_components(dtype=torch.bfloat16) | |
| pipe.enable_safety_checker() | |
| <span class="hljs-keyword">if</span> tp_degree > <span class="hljs-number">1</span>: | |
| enable_cosmos3_tensor_parallel(pipe.transformer, mesh[<span class="hljs-string">"tp"</span>]) <span class="hljs-comment"># shard weights -> GPUs</span> | |
| pipe.to(<span class="hljs-string">f"cuda:<span class="hljs-subst">{local_rank}</span>"</span>) <span class="hljs-comment"># move the replicated remainder</span> | |
| pipe.transformer.set_attention_backend(<span class="hljs-string">"native"</span>) | |
| <span class="hljs-keyword">if</span> cp_degree > <span class="hljs-number">1</span>: | |
| enable_cosmos3_context_parallel(pipe.transformer, mesh[<span class="hljs-string">"cp"</span>]) <span class="hljs-comment"># shard the sequence</span> | |
| <span class="hljs-keyword">elif</span> tp_degree > <span class="hljs-number">1</span>: | |
| enable_cosmos3_flash_attention(pipe.transformer) <span class="hljs-comment"># GQA-safe dense attention</span> | |
| <span class="hljs-comment"># Modular pipelines replace components through update_components().</span> | |
| scheduler = UniPCMultistepScheduler.from_config( | |
| pipe.scheduler.config, flow_shift=<span class="hljs-number">10.0</span>, use_karras_sigmas=<span class="hljs-literal">False</span> | |
| ) | |
| pipe.update_components(scheduler=scheduler) | |
| <span class="hljs-comment"># A single output name returns that value; a list returns a dictionary.</span> | |
| outputs = pipe( | |
| prompt=<span class="hljs-string">'{"scene":"A robot arm in a kitchen"}'</span>, | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| output=[<span class="hljs-string">"videos"</span>, <span class="hljs-string">"sound"</span>, <span class="hljs-string">"sampling_rate"</span>, <span class="hljs-string">"action"</span>], | |
| ) | |
| videos = outputs[<span class="hljs-string">"videos"</span>] | |
| sound = outputs[<span class="hljs-string">"sound"</span>] <span class="hljs-comment"># None unless sound generation was requested.</span> | |
| action = outputs[<span class="hljs-string">"action"</span>] <span class="hljs-comment"># None unless an action workflow produced actions.</span><!----></pre></div><!----> <p>Pour CP uniquement (sans partage de poids), utilisez un maillage 1-D : <code>init_device_mesh("cuda", (world_size,), mesh_dim_names=("cp",))</code> et simplement <code>enable_cosmos3_context_parallel</code>.</p> <p><code>enable_safety_checker()</code> charge et active le vérificateur par défaut ; <code>disable_safety_checker()</code> le désactive explicitement. Utilisez ces méthodes de pipeline plutôt que l’argument de construction <code>enable_safety_checker=</code> du pipeline de tâche ou l’argument d’appel <code>enable_safety_check=</code>. Les pipelines modulaires ne renvoient pas non plus <code>Cosmos3OmniPipelineOutput</code> : utilisez <code>output="videos"</code> pour les seules images, ou une liste de sorties et son dictionnaire renvoyé comme indiqué ci-dessus, au lieu de <code>result.video</code>, <code>result.sound</code> ou <code>result.action</code>.</p> <blockquote class="tip"><p>Sur certaines topologies multi-GPU, le premier all-to-all NCCL peut bloquer. Si une exécution CP stagne au début de la première étape de débruitage, définissez <code>NCCL_P2P_DISABLE=1</code> dans l’environnement avant de lancer <code>torchrun</code>.</p></blockquote> <p>CP et TP se combinent avec tous les flux de travail ci-dessus (texte-vers-vidéo, image-vers-vidéo, texte-vers-vidéo avec son, et génération conditionnée par l'action) ainsi qu'avec les points de contrôle <code>Nano</code> et <code>Super</code> — seules les lignes de construction du pipeline et de configuration du parallélisme changent.</p> <!--[1--><h2 class="relative group"><a id="metadata-templates" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#metadata-templates"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Modèles de métadonnées<!----></span></h2><!--]--><!----> <p><code>tokenize_prompt</code> ajoute de courtes phrases de métadonnées dans le message utilisateur afin que le LLM perçoive les conditions avec lesquelles le modèle a été entraîné. Le prompt positif reçoit des phrases comme <em>« La vidéo dure 7,9 secondes et est de 24 FPS. »</em> et <em>« Cette vidéo est d’une résolution de 720x1280. »</em> ; le prompt négatif reçoit l’inverse (<em>« … n’est pas … »</em>).</p> <p>Les deux sont activés par défaut. Désactivez l'une ou l'autre paire via <code>__call__</code> :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copié</div><!----></button><!----></div> <pre class="language-python "><!---->result = pipe( | |
| prompt=prompt, | |
| negative_prompt=negative_prompt, | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| fps=<span class="hljs-number">24.0</span>, | |
| add_duration_template=<span class="hljs-literal">False</span>, <span class="hljs-comment"># skip the duration sentence on both prompts</span> | |
| add_resolution_template=<span class="hljs-literal">False</span>, <span class="hljs-comment"># skip the resolution sentence on both prompts</span> | |
| )<!----></pre></div><!----> <p><code>add_duration_template</code> n'a aucun effet lorsque <code>num_frames == 1</code> (mode image) ; dans ce cas, seule la phrase de résolution est ajoutée.</p> <!--[1--><h2 class="relative group"><a id="safety-checker" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#safety-checker"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Vérificateur de sécurité<!----></span></h2><!--]--><!----> <p>Cosmos3 intègre le <a href="https://pypi.org/project/cosmos-guardrail/" rel="nofollow"><code>cosmos_guardrail</code></a> <code>CosmosSafetyChecker</code> et l'exécute <strong>par défaut</strong>. Le garde-fou texte rejette les invites non sûres avant la génération (<code>ValueError</code>) ; le garde-fou vidéo s'exécute sur les images décodées et applique soit un floutage des visages détectés, soit un rejet de la sortie. La sortie audio n'est pas protégée par un garde-fou.</p> <p>Installez la dépendance facultative pour activer le vérificateur par défaut :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copié</div><!----></button><!----></div> <pre class=" "><!---->pip <span class="hljs-keyword">install</span> cosmos_guardrail<!----></pre></div><!----> <p>Le vérificateur est obligatoire dans le cadre de l’accord de licence des modèles ouverts NVIDIA. Les deux drapeaux ci-dessous existent pour les flux de travail de test et de développement où la garde-fou serait redondante (par exemple, si l’entrée a déjà été nettoyée, ou si vous exercez intentionnellement le pipeline sur des entrées limites).</p> <p><strong>Désactiver à la construction</strong> (aucun vérificateur n'est instancié, donc aucun modèle de garde-fou n'est téléchargé ni chargé en mémoire) :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copié</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniPipeline | |
| pipe = Cosmos3OmniPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Nano"</span>, | |
| dtype=torch.bfloat16, | |
| device_map=<span class="hljs-string">"cuda"</span>, <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| enable_safety_checker=<span class="hljs-literal">False</span>, | |
| )<!----></pre></div><!----> <p><strong>Désactiver pour un seul appel</strong> (le vérificateur reste chargé — utile pour un contournement ponctuel tout en conservant l’activation par défaut pour les appels suivants) :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copié</div><!----></button><!----></div> <pre class="language-python "><!---->result = pipe( | |
| prompt=prompt, | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| fps=<span class="hljs-number">24.0</span>, | |
| enable_safety_check=<span class="hljs-literal">False</span>, | |
| )<!----></pre></div><!----> <p>Pour fournir un vérificateur personnalisé (par exemple, une sous-classe sans opération pour des tests rapides), passez-le en tant que <code>safety_checker=</code> :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copié</div><!----></button><!----></div> <pre class="language-python "><!---->pipe = Cosmos3OmniPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Nano"</span>, | |
| dtype=torch.bfloat16, | |
| device_map=<span class="hljs-string">"cuda"</span>, <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| safety_checker=MyCustomSafetyChecker(), | |
| )<!----></pre></div><!----> <!--[1--><h2 class="relative group"><a id="diffusers.Cosmos3OmniPipeline" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#diffusers.Cosmos3OmniPipeline"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Cosmos3OmniPipeline<!----></span></h2><!--]--><!----> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><div><span class="group flex space-x-1.5 items-center text-gray-800 bg-gradient-to-r rounded-tr-lg -mt-4 -ml-4 pt-3 px-2.5" id="diffusers.Cosmos3OmniPipeline"><!----><h3 class="!m-0"><span class="flex-1 break-all md:text-lg bg-gradient-to-r px-2.5 py-1.5 rounded-xl from-indigo-50/70 to-white dark:from-gray-900 dark:to-gray-950 dark:text-indigo-300 text-indigo-700"><svg class="mr-1.5 text-indigo-500 inline-block -mt-0.5" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width=".8em" height=".8em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 24 24"><path class="uim-quaternary" d="M20.23 7.24L12 12L3.77 7.24a1.98 1.98 0 0 1 .7-.71L11 2.76c.62-.35 1.38-.35 2 0l6.53 3.77c.29.173.531.418.7.71z" opacity=".25" fill="currentColor"></path><path class="uim-tertiary" d="M12 12v9.5a2.09 2.09 0 0 1-.91-.21L4.5 17.48a2.003 2.003 0 0 1-1-1.73v-7.5a2.06 2.06 0 0 1 .27-1.01L12 12z" opacity=".5" fill="currentColor"></path><path class="uim-primary" d="M20.5 8.25v7.5a2.003 2.003 0 0 1-1 1.73l-6.62 3.82c-.275.13-.576.198-.88.2V12l8.23-4.76c.175.308.268.656.27 1.01z" fill="currentColor"></path></svg><span class="font-light">classe</span> <span class="font-medium">diffusers.</span><span class="font-semibold">Cosmos3OmniPipeline</span></span></h3><!----> <a id="diffusers.Cosmos3OmniPipeline" class="header-link invisible with-hover:group-hover:visible pr-2" href="#diffusers.Cosmos3OmniPipeline"><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></a> <!--[0--><a class="!ml-auto !text-gray-400 !no-underline text-sm flex items-center" href="https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/cosmos/pipeline_cosmos3_omni.py#L413" target="_blank"><span><</span> <span class="hidden md:block mx-0.5 hover:!underline">source</span> <span>></span></a><!--]--></span> <!--[0--><p class="font-mono text-xs md:text-sm !leading-relaxed !my-6"><span>(</span> <!--[--><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">transformer<span class="opacity-60">: Cosmos3OmniTransformer</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">text_tokenizer<span class="opacity-60">: AutoTokenizer</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">vae<span class="opacity-60">: AutoencoderKLWan</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">scheduler<span class="opacity-60">: UniPCMultistepScheduler</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">sound_tokenizer<span class="opacity-60">: diffusers.models.autoencoders.autoencoder_cosmos3_audio.Cosmos3AVAEAudioTokenizer | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">safety_checker<span class="opacity-60">: diffusers.pipelines.cosmos.pipeline_cosmos3_omni.CosmosSafetyChecker | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">enable_safety_checker<span class="opacity-60">: bool = True</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">default_use_system_prompt<span class="opacity-60">: bool = True</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">use_native_flow_schedule<span class="opacity-60">: bool = False</span></span></span><!--]--> <span>)</span> <!--[-1--><!--]--></p><!--]--> <div class="!mb-10 relative docstring-details "><!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--></div></div><!----> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><div><span class="group flex space-x-1.5 items-center text-gray-800 bg-gradient-to-r rounded-tr-lg -mt-4 -ml-4 pt-3 px-2.5" id="diffusers.Cosmos3OmniPipeline.decode_sound"><!----><h4 class="!m-0"><span class="flex-1 rounded-xl py-0.5 break-all bg-gradient-to-r from-blue-50/60 to-white dark:from-gray-900 dark:to-gray-950 text-blue-700 dark:text-blue-300 font-medium px-2"><svg width="1em" height="1em" viewBox="0 0 32 33" class="mr-1 inline-block -mt-0.5" xmlns="http://www.w3.org/2000/svg"><path d="M5.80566 18.3545C4.90766 17.4565 4.90766 16.0005 5.80566 15.1025L14.3768 6.53142C15.2748 5.63342 16.7307 5.63342 17.6287 6.53142L26.1999 15.1025C27.0979 16.0005 27.0979 17.4565 26.1999 18.3545L17.6287 26.9256C16.7307 27.8236 15.2748 27.8236 14.3768 26.9256L5.80566 18.3545Z" fill="currentColor" fill-opacity="0.25"/><path fill-rule="evenodd" clip-rule="evenodd" d="M16.4801 13.9619C16.4801 12.9761 16.7467 12.5436 16.9443 12.3296C17.1764 12.078 17.5731 11.8517 18.2275 11.707C18.8821 11.5623 19.638 11.5342 20.4038 11.5582C20.7804 11.57 21.1341 11.5932 21.4719 11.6156L21.5263 11.6193C21.8195 11.6389 22.1626 11.6618 22.4429 11.6618V7.40825C22.3209 7.40825 22.1219 7.39596 21.7544 7.37149C21.4202 7.34925 20.9976 7.32115 20.5371 7.30672C19.6286 7.27824 18.4672 7.29779 17.3093 7.55377C16.1512 7.8098 14.8404 8.33724 13.8181 9.4452C12.7612 10.5907 12.2266 12.1236 12.2266 13.9619V15.0127H10.6836V19.2662H12.2266V26.6332H16.4801V19.2662H20.3394V15.0127H16.4801V13.9619Z" fill="currentColor"/></svg>decode_sound</span></h4><!----> <a id="diffusers.Cosmos3OmniPipeline.decode_sound" class="header-link invisible with-hover:group-hover:visible pr-2" href="#diffusers.Cosmos3OmniPipeline.decode_sound"><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></a> <!--[0--><a class="!ml-auto !text-gray-400 !no-underline text-sm flex items-center" href="https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/cosmos/pipeline_cosmos3_omni.py#L517" target="_blank"><span><</span> <span class="hidden md:block mx-0.5 hover:!underline">source</span> <span>></span></a><!--]--></span> <!--[0--><p class="font-mono text-xs md:text-sm !leading-relaxed !my-6"><span>(</span> <!--[--><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">latent<span class="opacity-60">: Tensor</span></span></span><!--]--> <span>)</span> <!--[-1--><!--]--></p><!--]--> <div class="!mb-10 relative docstring-details "><!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--></div></div><!----> <p>Décode un latent sonore <code>[C, T]</code> en une forme d'onde <code>[audio_ch, N]</code>.</p> <p>Ajoute/supprime la dimension de lot attendue par le décodeur du tokenizer audio.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><div><span class="group flex space-x-1.5 items-center text-gray-800 bg-gradient-to-r rounded-tr-lg -mt-4 -ml-4 pt-3 px-2.5" id="diffusers.Cosmos3OmniPipeline.prepare_latents"><!----><h4 class="!m-0"><span class="flex-1 rounded-xl py-0.5 break-all bg-gradient-to-r from-blue-50/60 to-white dark:from-gray-900 dark:to-gray-950 text-blue-700 dark:text-blue-300 font-medium px-2"><svg width="1em" height="1em" viewBox="0 0 32 33" class="mr-1 inline-block -mt-0.5" xmlns="http://www.w3.org/2000/svg"><path d="M5.80566 18.3545C4.90766 17.4565 4.90766 16.0005 5.80566 15.1025L14.3768 6.53142C15.2748 5.63342 16.7307 5.63342 17.6287 6.53142L26.1999 15.1025C27.0979 16.0005 27.0979 17.4565 26.1999 18.3545L17.6287 26.9256C16.7307 27.8236 15.2748 27.8236 14.3768 26.9256L5.80566 18.3545Z" fill="currentColor" fill-opacity="0.25"/><path fill-rule="evenodd" clip-rule="evenodd" d="M16.4801 13.9619C16.4801 12.9761 16.7467 12.5436 16.9443 12.3296C17.1764 12.078 17.5731 11.8517 18.2275 11.707C18.8821 11.5623 19.638 11.5342 20.4038 11.5582C20.7804 11.57 21.1341 11.5932 21.4719 11.6156L21.5263 11.6193C21.8195 11.6389 22.1626 11.6618 22.4429 11.6618V7.40825C22.3209 7.40825 22.1219 7.39596 21.7544 7.37149C21.4202 7.34925 20.9976 7.32115 20.5371 7.30672C19.6286 7.27824 18.4672 7.29779 17.3093 7.55377C16.1512 7.8098 14.8404 8.33724 13.8181 9.4452C12.7612 10.5907 12.2266 12.1236 12.2266 13.9619V15.0127H10.6836V19.2662H12.2266V26.6332H16.4801V19.2662H20.3394V15.0127H16.4801V13.9619Z" fill="currentColor"/></svg>prepare_latents</span></h4><!----> <a id="diffusers.Cosmos3OmniPipeline.prepare_latents" class="header-link invisible with-hover:group-hover:visible pr-2" href="#diffusers.Cosmos3OmniPipeline.prepare_latents"><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></a> <!--[0--><a class="!ml-auto !text-gray-400 !no-underline text-sm flex items-center" href="https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/cosmos/pipeline_cosmos3_omni.py#L763" target="_blank"><span><</span> <span class="hidden md:block mx-0.5 hover:!underline">source</span> <span>></span></a><!--]--></span> <!--[0--><p class="font-mono text-xs md:text-sm !leading-relaxed !my-6"><span>(</span> <!--[--><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">image<span class="opacity-60">: typing.Union[PIL.Image.Image, numpy.ndarray, torch.Tensor, NoneType] = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">video<span class="opacity-60">: typing.Union[list[PIL.Image.Image], torch.Tensor, numpy.ndarray, NoneType] = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">condition_frame_indexes_vision<span class="opacity-60">: collections.abc.Iterable[int] = (0, 1)</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">condition_video_keep<span class="opacity-60">: typing.Literal['first', 'last'] = 'first'</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">num_frames<span class="opacity-60">: int | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">height<span class="opacity-60">: int | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">width<span class="opacity-60">: int | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">fps<span class="opacity-60">: float = 24.0</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">latents<span class="opacity-60">: typing.Optional[torch.Tensor] = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">sound_latents<span class="opacity-60">: typing.Optional[torch.Tensor] = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">action_latents<span class="opacity-60">: typing.Optional[torch.Tensor] = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">generator<span class="opacity-60">: typing.Optional[torch.Generator] = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">device<span class="opacity-60">: str = 'cuda'</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">dtype<span class="opacity-60">: dtype = torch.bfloat16</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">enable_sound<span class="opacity-60">: bool = False</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">action<span class="opacity-60">: CosmosActionCondition | None = None</span></span></span><!--]--> <span>)</span> <!--[-1--><!--]--></p><!--]--> <div class="!mb-10 relative docstring-details "><!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--></div></div><!----> <p>Construire le conditionnement et le bruit initial pour un seul échantillon.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><div><span class="group flex space-x-1.5 items-center text-gray-800 bg-gradient-to-r rounded-tr-lg -mt-4 -ml-4 pt-3 px-2.5" id="diffusers.Cosmos3OmniPipeline.tokenize_prompt"><!----><h4 class="!m-0"><span class="flex-1 rounded-xl py-0.5 break-all bg-gradient-to-r from-blue-50/60 to-white dark:from-gray-900 dark:to-gray-950 text-blue-700 dark:text-blue-300 font-medium px-2"><svg width="1em" height="1em" viewBox="0 0 32 33" class="mr-1 inline-block -mt-0.5" xmlns="http://www.w3.org/2000/svg"><path d="M5.80566 18.3545C4.90766 17.4565 4.90766 16.0005 5.80566 15.1025L14.3768 6.53142C15.2748 5.63342 16.7307 5.63342 17.6287 6.53142L26.1999 15.1025C27.0979 16.0005 27.0979 17.4565 26.1999 18.3545L17.6287 26.9256C16.7307 27.8236 15.2748 27.8236 14.3768 26.9256L5.80566 18.3545Z" fill="currentColor" fill-opacity="0.25"/><path fill-rule="evenodd" clip-rule="evenodd" d="M16.4801 13.9619C16.4801 12.9761 16.7467 12.5436 16.9443 12.3296C17.1764 12.078 17.5731 11.8517 18.2275 11.707C18.8821 11.5623 19.638 11.5342 20.4038 11.5582C20.7804 11.57 21.1341 11.5932 21.4719 11.6156L21.5263 11.6193C21.8195 11.6389 22.1626 11.6618 22.4429 11.6618V7.40825C22.3209 7.40825 22.1219 7.39596 21.7544 7.37149C21.4202 7.34925 20.9976 7.32115 20.5371 7.30672C19.6286 7.27824 18.4672 7.29779 17.3093 7.55377C16.1512 7.8098 14.8404 8.33724 13.8181 9.4452C12.7612 10.5907 12.2266 12.1236 12.2266 13.9619V15.0127H10.6836V19.2662H12.2266V26.6332H16.4801V19.2662H20.3394V15.0127H16.4801V13.9619Z" fill="currentColor"/></svg>tokenize_prompt</span></h4><!----> <a id="diffusers.Cosmos3OmniPipeline.tokenize_prompt" class="header-link invisible with-hover:group-hover:visible pr-2" href="#diffusers.Cosmos3OmniPipeline.tokenize_prompt"><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></a> <!--[0--><a class="!ml-auto !text-gray-400 !no-underline text-sm flex items-center" href="https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/cosmos/pipeline_cosmos3_omni.py#L1132" target="_blank"><span><</span> <span class="hidden md:block mx-0.5 hover:!underline">source</span> <span>></span></a><!--]--></span> <!--[0--><p class="font-mono text-xs md:text-sm !leading-relaxed !my-6"><span>(</span> <!--[--><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">prompt<span class="opacity-60">: str</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">negative_prompt<span class="opacity-60">: str | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">num_frames<span class="opacity-60">: int = 189</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">height<span class="opacity-60">: int = 720</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">width<span class="opacity-60">: int = 1280</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">fps<span class="opacity-60">: float = 24.0</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">use_system_prompt<span class="opacity-60">: bool | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">add_resolution_template<span class="opacity-60">: bool = True</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">add_duration_template<span class="opacity-60">: bool = True</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">action_mode<span class="opacity-60">: str | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">action_view_point<span class="opacity-60">: str | None = None</span></span></span><!--]--> <span>)</span> <!--[-1--><!--]--></p><!--]--> <div class="!mb-10 relative docstring-details "><!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--></div></div><!----> <p>Appliquez les modèles d'augmentation de prompt et tokenisez les prompts cond/uncond via le modèle de chat configuré.</p> <p>Ce pipeline n'exécute pas d'encodeur de texte séparé : le transformeur joint Cosmos3 consomme les identifiants de jetons bruts en même temps que les jetons visuels (et, le cas échéant, sonores).</p> <p>Lorsque <code>negative_prompt</code> est <code>None</code>, une chaîne vide est utilisée ; la page de documentation de Cosmos3 documente les prompts négatifs de contrôle qualité recommandés à passer explicitement pour text2video / image2video. Les modèles de durée et de résolution sont ajoutés au prompt, et les modèles inverses sont ajoutés au prompt négatif, lorsque cette option est activée.</p> <p>Lorsque <code>action_mode</code> est défini, le prompt est plutôt converti en la légende JSON d’action structurée sur laquelle le modèle a été entraîné (voir <code>_build_action_json_prompt</code>), en utilisant <code>action_view_point</code> pour le champ de cadrage ; les modèles de métadonnées plates sont ignorés car le JSON contient déjà duration/fps/resolution/aspect_ratio.</p></div></div> <ul><li>tous</li> <li><strong>appel</strong></li></ul> <!--[1--><h2 class="relative group"><a id="cosmos3omnimodularpipeline" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#cosmos3omnimodularpipeline"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Cosmos3OmniModularPipeline<!----></span></h2><!--]--><!----> <p>Cosmos 3 est également disponible en tant que pipeline Modular Diffusers. Le pipeline basé sur les tâches <a href="/docs/diffusers/main/fr/api/pipelines/cosmos3#diffusers.Cosmos3OmniPipeline">Cosmos3OmniPipeline</a> reste disponible ; le pipeline modulaire coexiste avec lui et couvre les mêmes modes (<code>text2image</code>, <code>text2video</code>, <code>image2video</code>, <code>video2video</code>, génération conditionnée par l'action et <code>transfer</code> (contrôle structurel), avec un son optionnel lorsque le checkpoint le prend en charge).</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copié</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniModularPipeline | |
| pipe = Cosmos3OmniModularPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Nano"</span>, dtype=torch.bfloat16 | |
| ) | |
| pipe.load_components(dtype=torch.bfloat16) | |
| pipe.enable_safety_checker() | |
| videos = pipe( | |
| prompt=<span class="hljs-string">'{"scene":"A robot arm in a kitchen"}'</span>, | |
| num_frames=<span class="hljs-number">1</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| output=<span class="hljs-string">"videos"</span>, | |
| ) | |
| <span class="hljs-comment"># Modular pipelines expose declared outputs directly instead of using the task pipeline's</span> | |
| <span class="hljs-comment"># `return_dict`/`Cosmos3OmniPipelineOutput` API.</span> | |
| image = videos[<span class="hljs-number">0</span>]<!----></pre></div><!----> <p>Vous pouvez également charger via <a href="/docs/diffusers/main/fr/api/modular_diffusers/pipeline#diffusers.ModularPipeline">ModularPipeline</a> et laisser la configuration du dépôt sélectionner la classe de blocs :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copié</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> ModularPipeline | |
| pipe = ModularPipeline.from_pretrained(<span class="hljs-string">"nvidia/Cosmos3-Nano"</span>, dtype=torch.bfloat16) | |
| pipe.load_components(dtype=torch.bfloat16) | |
| pipe.enable_safety_checker() | |
| videos = pipe( | |
| prompt=<span class="hljs-string">'{"scene":"A robot arm in a kitchen"}'</span>, num_frames=<span class="hljs-number">1</span>, height=<span class="hljs-number">720</span>, width=<span class="hljs-number">1280</span>, output=<span class="hljs-string">"videos"</span> | |
| )<!----></pre></div><!----> <p>Pour inspecter ou personnaliser un flux de travail modulaire Cosmos spécifique, utilisez <code>available_workflows</code> + <code>get_workflow()</code> :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copié</div><!----></button><!----></div> <pre class="language-python "><!---->available = pipe.blocks.available_workflows | |
| image2video_blocks = pipe.blocks.get_workflow(<span class="hljs-string">"image2video"</span>)<!----></pre></div><!----> <!--[2--><h3 class="relative group"><a id="modular-examples-for-all-existing-workflows" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#modular-examples-for-all-existing-workflows"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Exemples modulaires pour tous les flux de travail existants<!----></span></h3><!--]--><!----> <p>Le pipeline modulaire prend en charge les mêmes signatures d’appel que le pipeline de tâche. Les extraits ci-dessous reproduisent chaque exemple de génération présenté plus haut (<code>text2video</code>, <code>text2image</code>, <code>image2video</code>, <code>video2video</code>, <code>video2video_sound</code>, <code>text2video_sound</code> et <code>action_policy</code>). Le transfert (contrôle structurel) possède ses propres entrées et est présenté séparément dans <a href="#modular-transfer-structural-control">Transfert modulaire</a> ci-dessous.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copié</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniModularPipeline, CosmosActionCondition | |
| <span class="hljs-keyword">from</span> diffusers.schedulers.scheduling_unipc_multistep <span class="hljs-keyword">import</span> UniPCMultistepScheduler | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> encode_video, export_to_video, load_image, load_video | |
| pipe = Cosmos3OmniModularPipeline.from_pretrained(<span class="hljs-string">"nvidia/Cosmos3-Nano"</span>, dtype=torch.bfloat16) | |
| pipe.load_components(dtype=torch.bfloat16) | |
| pipe.enable_safety_checker() | |
| pipe.to(<span class="hljs-string">"cuda"</span>) <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| pipe.scheduler = UniPCMultistepScheduler.from_config( | |
| pipe.scheduler.config, flow_shift=<span class="hljs-number">10.0</span>, use_karras_sigmas=<span class="hljs-literal">False</span> | |
| ) | |
| <span class="hljs-comment"># text2video</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_t2v_prompt.json"</span>)) | |
| negative_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/negative_prompt.json"</span>)) | |
| videos = pipe( | |
| prompt=json.dumps(json_prompt), | |
| negative_prompt=json.dumps(negative_prompt), | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| num_inference_steps=<span class="hljs-number">35</span>, | |
| guidance_scale=<span class="hljs-number">6.0</span>, | |
| fps=<span class="hljs-number">24.0</span>, | |
| output=<span class="hljs-string">"videos"</span>, | |
| ) | |
| export_to_video(videos, <span class="hljs-string">"cosmos3_modular_t2v.mp4"</span>, fps=<span class="hljs-number">24</span>, macro_block_size=<span class="hljs-number">1</span>) | |
| <span class="hljs-comment"># text2image</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_t2i_prompt.json"</span>)) | |
| videos = pipe(prompt=json.dumps(json_prompt), num_frames=<span class="hljs-number">1</span>, height=<span class="hljs-number">720</span>, width=<span class="hljs-number">1280</span>, output=<span class="hljs-string">"videos"</span>) | |
| videos[<span class="hljs-number">0</span>].save(<span class="hljs-string">"cosmos3_modular_t2i.jpg"</span>, <span class="hljs-built_in">format</span>=<span class="hljs-string">"JPEG"</span>, quality=<span class="hljs-number">85</span>) | |
| <span class="hljs-comment"># image2video</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_i2v_prompt.json"</span>)) | |
| negative_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/negative_prompt_i2v.json"</span>)) | |
| image = load_image(<span class="hljs-string">"https://github.com/nvidia-cosmos/cosmos-dependencies/releases/download/assets/robot_153.jpg"</span>) | |
| videos = pipe( | |
| prompt=json.dumps(json_prompt), | |
| negative_prompt=json.dumps(negative_prompt), | |
| image=image, | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| fps=<span class="hljs-number">24.0</span>, | |
| output=<span class="hljs-string">"videos"</span>, | |
| ) | |
| export_to_video(videos, <span class="hljs-string">"cosmos3_modular_i2v.mp4"</span>, fps=<span class="hljs-number">24</span>, macro_block_size=<span class="hljs-number">1</span>) | |
| <span class="hljs-comment"># video2video</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_v2v_prompt.json"</span>)) | |
| negative_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/negative_prompt_i2v.json"</span>)) | |
| video = load_video( | |
| <span class="hljs-string">"https://github.com/nvidia-cosmos/cosmos-dependencies/raw/refs/heads/assets/cosmos3/inputs/vision/robot_pouring.mp4"</span> | |
| ) | |
| videos = pipe( | |
| prompt=json.dumps(json_prompt), | |
| negative_prompt=json.dumps(negative_prompt), | |
| video=video, | |
| condition_frame_indexes_vision=[<span class="hljs-number">0</span>, <span class="hljs-number">1</span>], | |
| condition_video_keep=<span class="hljs-string">"first"</span>, | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| num_inference_steps=<span class="hljs-number">35</span>, | |
| guidance_scale=<span class="hljs-number">6.0</span>, | |
| fps=<span class="hljs-number">24.0</span>, | |
| output=<span class="hljs-string">"videos"</span>, | |
| ) | |
| export_to_video(videos, <span class="hljs-string">"cosmos3_modular_v2v.mp4"</span>, fps=<span class="hljs-number">24</span>, macro_block_size=<span class="hljs-number">1</span>) | |
| <span class="hljs-comment"># video2video_sound</span> | |
| outputs = pipe( | |
| prompt=json.dumps(json_prompt), | |
| negative_prompt=json.dumps(negative_prompt), | |
| video=video, | |
| condition_frame_indexes_vision=[<span class="hljs-number">0</span>, <span class="hljs-number">1</span>], | |
| condition_video_keep=<span class="hljs-string">"first"</span>, | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| fps=<span class="hljs-number">24.0</span>, | |
| enable_sound=<span class="hljs-literal">True</span>, | |
| output=[<span class="hljs-string">"videos"</span>, <span class="hljs-string">"sound"</span>, <span class="hljs-string">"sampling_rate"</span>], | |
| ) | |
| encode_video( | |
| outputs[<span class="hljs-string">"videos"</span>], | |
| fps=<span class="hljs-number">24</span>, | |
| audio=outputs[<span class="hljs-string">"sound"</span>], | |
| audio_sample_rate=outputs[<span class="hljs-string">"sampling_rate"</span>], | |
| output_path=<span class="hljs-string">"cosmos3_modular_v2v_with_sound.mp4"</span>, | |
| ) | |
| <span class="hljs-comment"># text2video_sound</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_t2v_sound_prompt.json"</span>)) | |
| negative_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/negative_prompt.json"</span>)) | |
| outputs = pipe( | |
| prompt=json.dumps(json_prompt), | |
| negative_prompt=json.dumps(negative_prompt), | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| fps=<span class="hljs-number">24.0</span>, | |
| enable_sound=<span class="hljs-literal">True</span>, | |
| output=[<span class="hljs-string">"videos"</span>, <span class="hljs-string">"sound"</span>, <span class="hljs-string">"sampling_rate"</span>], | |
| ) | |
| encode_video( | |
| outputs[<span class="hljs-string">"videos"</span>], | |
| fps=<span class="hljs-number">24</span>, | |
| audio=outputs[<span class="hljs-string">"sound"</span>], | |
| audio_sample_rate=outputs[<span class="hljs-string">"sampling_rate"</span>], | |
| output_path=<span class="hljs-string">"cosmos3_modular_t2v_with_sound.mp4"</span>, | |
| ) | |
| <span class="hljs-comment"># action_policy</span> | |
| prompt = <span class="hljs-string">"Put the pot to the left of the purple item."</span> | |
| action_video = load_video( | |
| <span class="hljs-string">"https://github.com/nvidia-cosmos/cosmos-dependencies/raw/refs/heads/assets/cosmos3/inputs/action/bridge_20260501_0.mp4"</span> | |
| ) | |
| outputs = pipe( | |
| prompt=prompt, | |
| action=CosmosActionCondition( | |
| mode=<span class="hljs-string">"policy"</span>, | |
| chunk_size=<span class="hljs-number">16</span>, | |
| domain_name=<span class="hljs-string">"bridge_orig_lerobot"</span>, | |
| resolution_tier=<span class="hljs-number">480</span>, | |
| video=action_video, | |
| view_point=<span class="hljs-string">"ego_view"</span>, | |
| ), | |
| fps=<span class="hljs-number">5</span>, | |
| num_inference_steps=<span class="hljs-number">30</span>, | |
| guidance_scale=<span class="hljs-number">1.0</span>, | |
| use_system_prompt=<span class="hljs-literal">False</span>, | |
| output=[<span class="hljs-string">"videos"</span>, <span class="hljs-string">"action"</span>], | |
| ) | |
| export_to_video(outputs[<span class="hljs-string">"videos"</span>], <span class="hljs-string">"cosmos3_modular_action_policy.mp4"</span>, fps=<span class="hljs-number">5</span>, macro_block_size=<span class="hljs-number">1</span>) | |
| <span class="hljs-keyword">if</span> outputs[<span class="hljs-string">"action"</span>] <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> <span class="hljs-literal">None</span>: | |
| <span class="hljs-keyword">with</span> <span class="hljs-built_in">open</span>(<span class="hljs-string">"cosmos3_modular_action_policy.json"</span>, <span class="hljs-string">"w"</span>) <span class="hljs-keyword">as</span> f: | |
| json.dump(outputs[<span class="hljs-string">"action"</span>][<span class="hljs-number">0</span>].tolist(), f)<!----></pre></div><!----> <!--[2--><h3 class="relative group"><a id="modular-transfer-structural-control" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#modular-transfer-structural-control"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Transfert modulaire (contrôle structurel)<!----></span></h3><!--]--><!----> <p>Le transfert suit une <strong>vidéo de contrôle précalculée</strong> (contour, flou, profondeur, segmentation ou carte de scénario de monde) transmise via <code>control_videos=</code> sous forme de mapping <code>{hint: video}</code>. Il est réservé aux vidéos (pas de <code>image</code> / <code>video</code> / <code>action</code> / <code>enable_sound</code>), le prompt est une légende JSON pré-échantillonnée (voir <a href="#prompt-upsampling">Échantillonnage du prompt</a>), et les clips longs sont générés de manière autorégressive par blocs de <code>num_video_frames_per_chunk</code> puis assemblés automatiquement. <code>guidance_scale</code> est le CFG texte habituel ; <code>control_guidance</code> (<code>!= 1.0</code>) amplifie en outre le signal de contrôle. Valeurs de départ recommandées par indice :</p> <table><thead><tr><th>Astuce</th><th><code>guidance_scale</code></th><th><code>control_guidance</code></th><th><code>flow_shift</code></th><th>Géométrie</th></tr></thead><tbody><tr><td>Bord / Flou / Profondeur</td><td>3.0</td><td>1.5</td><td>10.0</td><td>121 images @ 30 FPS</td></tr><tr><td>Segmentation</td><td>3.0</td><td>2.0</td><td>10.0</td><td>121 images @ 30 FPS</td></tr><tr><td>Scénario mondial (WSM)</td><td>1.0</td><td>3.0</td><td>10.0</td><td>101 images @ 10 i/s</td></tr></tbody></table> <p>Diffusers ne fournit pas les assets de contrôle. Des assets prêts à l’emploi (une vidéo de contrôle + un <code>prompt.json</code> correspondant par indice, ainsi qu’un <code>negative_prompt.json</code> partagé) sont disponibles dans le <a href="https://github.com/NVIDIA/cosmos/tree/6e2c69afafd6444c1ca14a021aba93fcc6b7996a/cookbooks/cosmos3/generator/transfer/assets" rel="nofollow">Cosmos cookbook</a>. Pour l’exemple de bordure ci-dessous, téléchargez-les dans un dossier local <code>assets/</code> :</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copié</div><!----></button><!----></div> <pre class="language-bash "><!---->base=https://github.com/NVIDIA/cosmos/raw/6e2c69afafd6444c1ca14a021aba93fcc6b7996a/cookbooks/cosmos3/generator/transfer/assets | |
| <span class="hljs-built_in">mkdir</span> -p assets/edge | |
| curl -sL <span class="hljs-string">"<span class="hljs-variable">$base</span>/edge/control_edge.mp4"</span> -o assets/edge/control_edge.mp4 | |
| curl -sL <span class="hljs-string">"<span class="hljs-variable">$base</span>/edge/prompt.json"</span> -o assets/edge/prompt.json | |
| curl -sL <span class="hljs-string">"<span class="hljs-variable">$base</span>/negative_prompt.json"</span> -o assets/negative_prompt.json<!----></pre></div><!----> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copié</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniModularPipeline | |
| <span class="hljs-keyword">from</span> diffusers.schedulers.scheduling_unipc_multistep <span class="hljs-keyword">import</span> UniPCMultistepScheduler | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video, load_video | |
| pipe = Cosmos3OmniModularPipeline.from_pretrained(<span class="hljs-string">"nvidia/Cosmos3-Nano"</span>, dtype=torch.bfloat16) | |
| pipe.load_components(dtype=torch.bfloat16) | |
| pipe.to(<span class="hljs-string">"cuda"</span>) <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| pipe.scheduler = UniPCMultistepScheduler.from_config( | |
| pipe.scheduler.config, flow_shift=<span class="hljs-number">10.0</span>, use_karras_sigmas=<span class="hljs-literal">False</span> | |
| ) | |
| <span class="hljs-comment"># Downloaded into assets/ from the Cosmos cookbook (see the curl snippet above).</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/edge/prompt.json"</span>)) | |
| negative_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/negative_prompt.json"</span>)) | |
| control_edge = load_video(<span class="hljs-string">"assets/edge/control_edge.mp4"</span>) | |
| videos = pipe( | |
| prompt=json.dumps(json_prompt), | |
| negative_prompt=json.dumps(negative_prompt), | |
| control_videos={<span class="hljs-string">"edge"</span>: control_edge}, | |
| num_frames=<span class="hljs-number">121</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| fps=<span class="hljs-number">30.0</span>, | |
| num_inference_steps=<span class="hljs-number">35</span>, | |
| guidance_scale=<span class="hljs-number">3.0</span>, | |
| control_guidance=<span class="hljs-number">1.5</span>, | |
| output=<span class="hljs-string">"videos"</span>, | |
| ) | |
| export_to_video(videos, <span class="hljs-string">"cosmos3_modular_transfer_edge.mp4"</span>, fps=<span class="hljs-number">30</span>, macro_block_size=<span class="hljs-number">1</span>)<!----></pre></div><!----> <!--[2--><h3 class="relative group"><a id="diffusers.Cosmos3OmniModularPipeline" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#diffusers.Cosmos3OmniModularPipeline"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Texte vers image et image vers vidéo distillés (peu d'étapes)<!----></span></h3><!--]--><!----> <p>Les points de contrôle distillés en peu d’étapes sont servis par <a href="/docs/diffusers/main/fr/api/pipelines/cosmos3#diffusers.Cosmos3DistilledModularPipeline">Cosmos3DistilledModularPipeline</a> (blocs : <code>Cosmos3DistilledBlocks</code>) ; les pipelines de base <a href="/docs/diffusers/main/fr/api/pipelines/cosmos3#diffusers.Cosmos3OmniModularPipeline">Cosmos3OmniModularPipeline</a> et <a href="/docs/diffusers/main/fr/api/pipelines/cosmos3#diffusers.Cosmos3OmniPipeline">Cosmos3OmniPipeline</a> ne les prennent pas en charge. <code>num_inference_steps</code> est fixé à la longueur de la configuration <code>distilled_sigmas</code> du pipeline (issue du <code>modular_model_index.json</code> du point de contrôle) et <code>guidance_scale</code> est forcé à 1.0, car le guidage est intégré dans les poids — passer toute autre valeur pour l’un ou l’autre provoque une erreur, et <code>negative_prompt</code> est signalé par un avertissement puis ignoré.</p> <p>Les points de contrôle distillés en FP8 (<code>revision="fp8"</code>) ne déclarent pas de politique de précision mixte, chaque étape reste donc en W8A8 natif.</p> <p>Les prompts suivent la même structure JSON descriptive que les modèles non distillés, il faut donc d’abord augmenter la résolution du texte court — utilisez <code>--mode text2image</code> (T2I) ou <code>--mode image2video</code> (I2V) comme décrit dans <a href="#prompt-upsampling">Augmentation de la résolution des prompts</a>, puis transmettez le JSON via <code>json.dumps(...)</code>.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copié</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3DistilledModularPipeline | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video, load_image | |
| <span class="hljs-comment"># JSON-upsampled prompt (see "Prompt upsampling" above).</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_t2i_prompt.json"</span>)) | |
| repo = <span class="hljs-string">"nvidia/Cosmos3-Super-Text2Image-4Step"</span> | |
| pipe = Cosmos3DistilledModularPipeline.from_pretrained(repo, dtype=torch.bfloat16) | |
| pipe.load_components(dtype=torch.bfloat16) | |
| pipe.to(<span class="hljs-string">"cuda"</span>) <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| <span class="hljs-comment"># text-to-image (distilled)</span> | |
| videos = pipe( | |
| prompt=json.dumps(json_prompt), | |
| num_frames=<span class="hljs-number">1</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| output=<span class="hljs-string">"videos"</span>, | |
| ) | |
| videos[<span class="hljs-number">0</span>].save(<span class="hljs-string">"cosmos3_distilled_t2i.jpg"</span>, <span class="hljs-built_in">format</span>=<span class="hljs-string">"JPEG"</span>, quality=<span class="hljs-number">85</span>) | |
| <span class="hljs-comment"># image-to-video (distilled) — load the I2V repo instead</span> | |
| <span class="hljs-comment"># JSON-upsampled prompt (see "Prompt upsampling" above); upsampled from the source prompt</span> | |
| <span class="hljs-comment"># "The right robotic hand picks up the red sphere on the shelf."</span> | |
| json_prompt_i2v = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_i2v_prompt.json"</span>)) | |
| repo_i2v = <span class="hljs-string">"nvidia/Cosmos3-Super-Image2Video-4Step"</span> | |
| pipe = Cosmos3DistilledModularPipeline.from_pretrained(repo_i2v, dtype=torch.bfloat16) | |
| pipe.load_components(dtype=torch.bfloat16) | |
| pipe.to(<span class="hljs-string">"cuda"</span>) | |
| image = load_image( | |
| <span class="hljs-string">"https://github.com/nvidia-cosmos/cosmos-dependencies/raw/refs/heads/assets/cosmos3/inputs/vision/robot_153.jpg"</span> | |
| ) | |
| videos = pipe( | |
| prompt=json.dumps(json_prompt_i2v), | |
| image=image, | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| output=<span class="hljs-string">"videos"</span>, | |
| ) | |
| export_to_video(videos, <span class="hljs-string">"cosmos3_distilled_i2v.mp4"</span>, fps=<span class="hljs-number">24</span>, macro_block_size=<span class="hljs-number">1</span>)<!----></pre></div><!----> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><div><span class="group flex space-x-1.5 items-center text-gray-800 bg-gradient-to-r rounded-tr-lg -mt-4 -ml-4 pt-3 px-2.5" id="diffusers.Cosmos3OmniModularPipeline"><!----><h3 class="!m-0"><span class="flex-1 break-all md:text-lg bg-gradient-to-r px-2.5 py-1.5 rounded-xl from-indigo-50/70 to-white dark:from-gray-900 dark:to-gray-950 dark:text-indigo-300 text-indigo-700"><svg class="mr-1.5 text-indigo-500 inline-block -mt-0.5" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width=".8em" height=".8em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 24 24"><path class="uim-quaternary" d="M20.23 7.24L12 12L3.77 7.24a1.98 1.98 0 0 1 .7-.71L11 2.76c.62-.35 1.38-.35 2 0l6.53 3.77c.29.173.531.418.7.71z" opacity=".25" fill="currentColor"></path><path class="uim-tertiary" d="M12 12v9.5a2.09 2.09 0 0 1-.91-.21L4.5 17.48a2.003 2.003 0 0 1-1-1.73v-7.5a2.06 2.06 0 0 1 .27-1.01L12 12z" opacity=".5" fill="currentColor"></path><path class="uim-primary" d="M20.5 8.25v7.5a2.003 2.003 0 0 1-1 1.73l-6.62 3.82c-.275.13-.576.198-.88.2V12l8.23-4.76c.175.308.268.656.27 1.01z" fill="currentColor"></path></svg><span class="font-light">classe</span> <span class="font-medium">diffusers.</span><span class="font-semibold">Cosmos3OmniModularPipeline</span></span></h3><!----> <a id="diffusers.Cosmos3OmniModularPipeline" class="header-link invisible with-hover:group-hover:visible pr-2" href="#diffusers.Cosmos3OmniModularPipeline"><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></a> <!--[0--><a class="!ml-auto !text-gray-400 !no-underline text-sm flex items-center" href="https://github.com/huggingface/diffusers/blob/main/src/diffusers/modular_pipelines/cosmos/modular_pipeline.py#L7" target="_blank"><span><</span> <span class="hidden md:block mx-0.5 hover:!underline">source</span> <span>></span></a><!--]--></span> <!--[0--><p class="font-mono text-xs md:text-sm !leading-relaxed !my-6"><span>(</span> <!--[--><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">blocks<span class="opacity-60">: diffusers.modular_pipelines.modular_pipeline.ModularPipelineBlocks | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">pretrained_model_name_or_path<span class="opacity-60">: str | os.PathLike | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">components_manager<span class="opacity-60">: diffusers.modular_pipelines.components_manager.ComponentsManager | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">collection<span class="opacity-60">: str | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">workflow<span class="opacity-60">: str | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">modular_config_dict<span class="opacity-60">: dict[str, typing.Any] | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">config_dict<span class="opacity-60">: dict[str, typing.Any] | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">**kwargs<span class="opacity-60"></span></span></span><!--]--> <span>)</span> <!--[-1--><!--]--></p><!--]--> <div class="!mb-10 relative docstring-details "><!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--></div></div><!----> <p>Un ModularPipeline pour la génération omni de Cosmos 3.</p></div> <ul><li>tous</li> <li><strong>appel</strong></li></ul> <!--[1--><h2 class="relative group"><a id="diffusers.Cosmos3DistilledModularPipeline" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#diffusers.Cosmos3DistilledModularPipeline"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Cosmos3DistilledModularPipeline<!----></span></h2><!--]--><!----> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><div><span class="group flex space-x-1.5 items-center text-gray-800 bg-gradient-to-r rounded-tr-lg -mt-4 -ml-4 pt-3 px-2.5" id="diffusers.Cosmos3DistilledModularPipeline"><!----><h3 class="!m-0"><span class="flex-1 break-all md:text-lg bg-gradient-to-r px-2.5 py-1.5 rounded-xl from-indigo-50/70 to-white dark:from-gray-900 dark:to-gray-950 dark:text-indigo-300 text-indigo-700"><svg class="mr-1.5 text-indigo-500 inline-block -mt-0.5" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width=".8em" height=".8em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 24 24"><path class="uim-quaternary" d="M20.23 7.24L12 12L3.77 7.24a1.98 1.98 0 0 1 .7-.71L11 2.76c.62-.35 1.38-.35 2 0l6.53 3.77c.29.173.531.418.7.71z" opacity=".25" fill="currentColor"></path><path class="uim-tertiary" d="M12 12v9.5a2.09 2.09 0 0 1-.91-.21L4.5 17.48a2.003 2.003 0 0 1-1-1.73v-7.5a2.06 2.06 0 0 1 .27-1.01L12 12z" opacity=".5" fill="currentColor"></path><path class="uim-primary" d="M20.5 8.25v7.5a2.003 2.003 0 0 1-1 1.73l-6.62 3.82c-.275.13-.576.198-.88.2V12l8.23-4.76c.175.308.268.656.27 1.01z" fill="currentColor"></path></svg><span class="font-light">classe</span> <span class="font-medium">diffusers.</span><span class="font-semibold">Cosmos3DistilledModularPipeline</span></span></h3><!----> <a id="diffusers.Cosmos3DistilledModularPipeline" class="header-link invisible with-hover:group-hover:visible pr-2" href="#diffusers.Cosmos3DistilledModularPipeline"><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></a> <!--[0--><a class="!ml-auto !text-gray-400 !no-underline text-sm flex items-center" href="https://github.com/huggingface/diffusers/blob/main/src/diffusers/modular_pipelines/cosmos/modular_pipeline.py#L121" target="_blank"><span><</span> <span class="hidden md:block mx-0.5 hover:!underline">source</span> <span>></span></a><!--]--></span> <!--[0--><p class="font-mono text-xs md:text-sm !leading-relaxed !my-6"><span>(</span> <!--[--><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">blocks<span class="opacity-60">: diffusers.modular_pipelines.modular_pipeline.ModularPipelineBlocks | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">pretrained_model_name_or_path<span class="opacity-60">: str | os.PathLike | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">components_manager<span class="opacity-60">: diffusers.modular_pipelines.components_manager.ComponentsManager | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">collection<span class="opacity-60">: str | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">workflow<span class="opacity-60">: str | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">modular_config_dict<span class="opacity-60">: dict[str, typing.Any] | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">config_dict<span class="opacity-60">: dict[str, typing.Any] | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">**kwargs<span class="opacity-60"></span></span></span><!--]--> <span>)</span> <!--[-1--><!--]--></p><!--]--> <div class="!mb-10 relative docstring-details "><!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--></div></div><!----> <p>Un ModularPipeline pour la génération omni de Cosmos 3 distillée (peu d'étapes).</p> <p>Les points de contrôle distillés intègrent le guidage sans classifieur dans les poids et échantillonnent selon un plan fixe lu depuis la configuration <code>distilled_sigmas</code> du pipeline (remplie à partir de <code>modular_model_index.json</code>), de sorte que <code>guidance_scale</code> et <code>num_inference_steps</code> sont fixes et que <code>negative_prompt</code> n’est pas pris en charge.</p></div> <ul><li>tous</li> <li><strong>appel</strong></li></ul> <!--[1--><h2 class="relative group"><a id="diffusers.CosmosActionCondition" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#diffusers.CosmosActionCondition"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->CosmosActionCondition<!----></span></h2><!--]--><!----> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><div><span class="group flex space-x-1.5 items-center text-gray-800 bg-gradient-to-r rounded-tr-lg -mt-4 -ml-4 pt-3 px-2.5" id="diffusers.CosmosActionCondition"><!----><h3 class="!m-0"><span class="flex-1 break-all md:text-lg bg-gradient-to-r px-2.5 py-1.5 rounded-xl from-indigo-50/70 to-white dark:from-gray-900 dark:to-gray-950 dark:text-indigo-300 text-indigo-700"><svg class="mr-1.5 text-indigo-500 inline-block -mt-0.5" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width=".8em" height=".8em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 24 24"><path class="uim-quaternary" d="M20.23 7.24L12 12L3.77 7.24a1.98 1.98 0 0 1 .7-.71L11 2.76c.62-.35 1.38-.35 2 0l6.53 3.77c.29.173.531.418.7.71z" opacity=".25" fill="currentColor"></path><path class="uim-tertiary" d="M12 12v9.5a2.09 2.09 0 0 1-.91-.21L4.5 17.48a2.003 2.003 0 0 1-1-1.73v-7.5a2.06 2.06 0 0 1 .27-1.01L12 12z" opacity=".5" fill="currentColor"></path><path class="uim-primary" d="M20.5 8.25v7.5a2.003 2.003 0 0 1-1 1.73l-6.62 3.82c-.275.13-.576.198-.88.2V12l8.23-4.76c.175.308.268.656.27 1.01z" fill="currentColor"></path></svg><span class="font-light">classe</span> <span class="font-medium">diffusers.</span><span class="font-semibold">CosmosActionCondition</span></span></h3><!----> <a id="diffusers.CosmosActionCondition" class="header-link invisible with-hover:group-hover:visible pr-2" href="#diffusers.CosmosActionCondition"><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></a> <!--[0--><a class="!ml-auto !text-gray-400 !no-underline text-sm flex items-center" href="https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/cosmos/pipeline_cosmos3_omni.py#L302" target="_blank"><span><</span> <span class="hidden md:block mx-0.5 hover:!underline">source</span> <span>></span></a><!--]--></span> <!--[0--><p class="font-mono text-xs md:text-sm !leading-relaxed !my-6"><span>(</span> <!--[--><span class="comma cursor-pointer"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">mode<span class="opacity-60">: typing.Literal['policy', 'forward_dynamics', 'inverse_dynamics']</span></span></span><span class="comma cursor-pointer"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">chunk_size<span class="opacity-60">: int</span></span></span><span class="comma cursor-pointer"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">domain_name<span class="opacity-60">: str</span></span></span><span class="comma cursor-pointer"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">resolution_tier<span class="opacity-60">: int = 480</span></span></span><span class="comma cursor-pointer"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">raw_actions<span class="opacity-60">: typing.Optional[torch.Tensor] = None</span></span></span><span class="comma cursor-pointer"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">image<span class="opacity-60">: typing.Union[PIL.Image.Image, numpy.ndarray, torch.Tensor, NoneType] = None</span></span></span><span class="comma cursor-pointer"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">video<span class="opacity-60">: typing.Union[list, numpy.ndarray, torch.Tensor, NoneType] = None</span></span></span><span class="comma cursor-pointer"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">view_point<span class="opacity-60">: str = 'ego_view'</span></span></span><!--]--> <span>)</span> <!--[-1--><!--]--></p><!--]--> <div class="!mb-10 relative docstring-details "><!--[-1--><!--]--> <!--[0--><p class="flex items-center font-semibold !mt-2 !mb-2 text-gray-800">Paramètres <span class="flex-auto border-t-2 border-gray-100 dark:border-gray-700 ml-3"></span></p> <ul class="px-2"><!--[--><li class="text-base !pl-4 my-3 rounded "><span class="group flex space-x-1.5 items-start"><a id="diffusers.CosmosActionCondition.mode" class="header-link block pr-0.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#diffusers.CosmosActionCondition.mode"><span><svg class="text-smd" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!----><strong>mode</strong> (<code>str</code>) — | |
| La tâche d’action. L’une des valeurs <code>"forward_dynamics"</code> (générer la vidéo future à partir d’une première image et d’une séquence de <code>raw_actions</code> donnée), <code>"inverse_dynamics"</code> (inférer les actions reliant les images de conditionnement) ou <code>"policy"</code> (générer conjointement la vidéo future et les actions à partir de la première image).<!----></span></span></li><li class="text-base !pl-4 my-3 rounded "><span class="group flex space-x-1.5 items-start"><a id="diffusers.CosmosActionCondition.chunk_size" class="header-link block pr-0.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#diffusers.CosmosActionCondition.chunk_size"><span><svg class="text-smd" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!----><strong>chunk_size</strong> (<code>int</code>) — | |
| Nombre d'étapes de transition d'action dans le chunk. La vidéo de conditionnement associée s'étend sur <code>chunk_size + 1</code> | |
| images.<!----></span></span></li><li class="text-base !pl-4 my-3 rounded "><span class="group flex space-x-1.5 items-start"><a id="diffusers.CosmosActionCondition.domain_name" class="header-link block pr-0.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#diffusers.CosmosActionCondition.domain_name"><span><svg class="text-smd" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!----><strong>domain_name</strong> (<code>str</code>) — | |
| Domaine d’incarnation sélectionnant les poids de projection d’action sensibles au domaine. Doit être l’un des domaines d’incarnation Cosmos 3 enregistrés. Il fixe également la largeur d’action non rembourrée utilisée pour extraire les actions prédites, résolue en interne à partir de ce nom (voir <code>_EMBODIMENT_TO_RAW_ACTION_DIM</code>).<!----></span></span></li><li class="text-base !pl-4 my-3 rounded "><span class="group flex space-x-1.5 items-start"><a id="diffusers.CosmosActionCondition.resolution_tier" class="header-link block pr-0.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#diffusers.CosmosActionCondition.resolution_tier"><span><svg class="text-smd" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!----><strong>resolution_tier</strong> (<code>int</code>, par défaut <code>480</code>) — | |
| Niveau de résolution <em>tier</em> pour le conditionnement de l’action (l’un des <code>256</code>, <code>480</code>, <code>704</code>, <code>720</code>). Le niveau sélectionne un canevas prédéfini dont le ratio d’aspect est le plus proche de celui de l’entrée ; l’entrée est réduite (jamais agrandie) et mise en bourre dans ce canevas pour le conditionnement. Il ne s’agit pas de la taille des images de sortie, qui suit le contenu de l’entrée. Faites correspondre le niveau à la résolution native de l’entrée : un niveau inférieur supprime des détails, tandis qu’un niveau supérieur n’ajoute aucune résolution (pas d’agrandissement), gaspille des calculs sur la bourre et constitue un désaccord entraînement/inférence susceptible de nuire à la qualité.<!----></span></span></li><li class="text-base !pl-4 my-3 rounded "><span class="group flex space-x-1.5 items-start"><a id="diffusers.CosmosActionCondition.raw_actions" class="header-link block pr-0.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#diffusers.CosmosActionCondition.raw_actions"><span><svg class="text-smd" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!----><strong>raw_actions</strong> (<code>torch.Tensor</code>, <em>facultatif</em>) — | |
| Vecteurs d’action bruts du domaine de forme <code>[T, raw_action_dim]</code> pilotant <code>"forward_dynamics"</code>. Les séquences plus courtes | |
| que <code>chunk_size</code> répètent la dernière action ; les plus longues sont tronquées. Les canaux au-delà de | |
| <code>action_dim</code> du modèle sont rejetés, et les entrées plus étroites sont complétées par des zéros jusqu’à <code>action_dim</code>.<!----></span></span></li><li class="text-base !pl-4 my-3 rounded "><span class="group flex space-x-1.5 items-start"><a id="diffusers.CosmosActionCondition.image" class="header-link block pr-0.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#diffusers.CosmosActionCondition.image"><span><svg class="text-smd" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!----><strong>image</strong> (<code>PIL.Image.Image</code>, <code>np.ndarray</code> ou <code>torch.Tensor</code>, <em>facultatif</em>) — | |
| Image de conditionnement pour <code>"policy"</code> / <code>"forward_dynamics"</code>. Mutuellement exclusif avec <code>video</code>.<!----></span></span></li><li class="text-base !pl-4 my-3 rounded "><span class="group flex space-x-1.5 items-start"><a id="diffusers.CosmosActionCondition.video" class="header-link block pr-0.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#diffusers.CosmosActionCondition.video"><span><svg class="text-smd" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!----><strong>video</strong> (<code>list</code>, <code>np.ndarray</code>, or <code>torch.Tensor</code>, <em>optional</em>) — | |
| Conditioning video, required for <code>"inverse_dynamics"</code>. For <code>"policy"</code> / <code>"forward_dynamics"</code> only its first | |
| frame is used. Mutually exclusive with <code>image</code>.<!----></span></span></li><li class="text-base !pl-4 my-3 rounded "><span class="group flex space-x-1.5 items-start"><a id="diffusers.CosmosActionCondition.view_point" class="header-link block pr-0.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#diffusers.CosmosActionCondition.view_point"><span><svg class="text-smd" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!----><strong>view_point</strong> (<code>str</code>, par défaut <code>"ego_view"</code>) — | |
| Étiquette de perspective de caméra utilisée pour remplir le champ <code>cinematography.framing</code> de la légende d’action. L’une des valeurs | |
| <code>"ego_view"</code>, <code>"third_person_view"</code>, <code>"wrist_view"</code> ou <code>"concat_view"</code>. Le modèle d’action a été entraîné sur | |
| des légendes JSON structurées qui contiennent cette phrase de point de vue ; une étiquette non reconnue supprime le champ framing | |
| (avec un avertissement).<!----></span></span></li><!--]--></ul><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--></div></div><!----> <p>Regroupe toutes les entrées requises pour une tâche de génération conditionnée par une action dans Cosmos 3.</p> <p>Passez ceci à <code>Cosmos3OmniPipeline.__call__()</code> via l'argument <code>action</code> plutôt que via les arguments de niveau supérieur <code>image</code> / <code>height</code> / <code>width</code>, qui sont réservés aux exécutions t2v et i2v.</p></div> <!--[1--><h2 class="relative group"><a id="diffusers.pipelines.cosmos.pipeline_cosmos3_omni.Cosmos3OmniPipelineOutput" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#diffusers.pipelines.cosmos.pipeline_cosmos3_omni.Cosmos3OmniPipelineOutput"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Cosmos3OmniPipelineOutput<!----></span></h2><!--]--><!----> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><div><span class="group flex space-x-1.5 items-center text-gray-800 bg-gradient-to-r rounded-tr-lg -mt-4 -ml-4 pt-3 px-2.5" id="diffusers.pipelines.cosmos.pipeline_cosmos3_omni.Cosmos3OmniPipelineOutput"><!----><h3 class="!m-0"><span class="flex-1 break-all md:text-lg bg-gradient-to-r px-2.5 py-1.5 rounded-xl from-indigo-50/70 to-white dark:from-gray-900 dark:to-gray-950 dark:text-indigo-300 text-indigo-700"><svg class="mr-1.5 text-indigo-500 inline-block -mt-0.5" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width=".8em" height=".8em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 24 24"><path class="uim-quaternary" d="M20.23 7.24L12 12L3.77 7.24a1.98 1.98 0 0 1 .7-.71L11 2.76c.62-.35 1.38-.35 2 0l6.53 3.77c.29.173.531.418.7.71z" opacity=".25" fill="currentColor"></path><path class="uim-tertiary" d="M12 12v9.5a2.09 2.09 0 0 1-.91-.21L4.5 17.48a2.003 2.003 0 0 1-1-1.73v-7.5a2.06 2.06 0 0 1 .27-1.01L12 12z" opacity=".5" fill="currentColor"></path><path class="uim-primary" d="M20.5 8.25v7.5a2.003 2.003 0 0 1-1 1.73l-6.62 3.82c-.275.13-.576.198-.88.2V12l8.23-4.76c.175.308.268.656.27 1.01z" fill="currentColor"></path></svg><span class="font-light">classe</span> <span class="font-medium">diffusers.pipelines.cosmos.pipeline_cosmos3_omni.</span><span class="font-semibold">Cosmos3OmniPipelineOutput</span></span></h3><!----> <a id="diffusers.pipelines.cosmos.pipeline_cosmos3_omni.Cosmos3OmniPipelineOutput" class="header-link invisible with-hover:group-hover:visible pr-2" href="#diffusers.pipelines.cosmos.pipeline_cosmos3_omni.Cosmos3OmniPipelineOutput"><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></a> <!--[0--><a class="!ml-auto !text-gray-400 !no-underline text-sm flex items-center" href="https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/cosmos/pipeline_cosmos3_omni.py#L283" target="_blank"><span><</span> <span class="hidden md:block mx-0.5 hover:!underline">source</span> <span>></span></a><!--]--></span> <!--[0--><p class="font-mono text-xs md:text-sm !leading-relaxed !my-6"><span>(</span> <!--[--><span class="comma cursor-pointer"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">video<span class="opacity-60">: Any</span></span></span><span class="comma cursor-pointer"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">sound<span class="opacity-60">: typing.Optional[torch.Tensor] = None</span></span></span><span class="comma cursor-pointer"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">action<span class="opacity-60">: list[torch.Tensor] | None = None</span></span></span><!--]--> <span>)</span> <!--[-1--><!--]--></p><!--]--> <div class="!mb-10 relative docstring-details "><!--[-1--><!--]--> <!--[0--><p class="flex items-center font-semibold !mt-2 !mb-2 text-gray-800">Paramètres <span class="flex-auto border-t-2 border-gray-100 dark:border-gray-700 ml-3"></span></p> <ul class="px-2"><!--[--><li class="text-base !pl-4 my-3 rounded "><span class="group flex space-x-1.5 items-start"><a id="diffusers.pipelines.cosmos.pipeline_cosmos3_omni.Cosmos3OmniPipelineOutput.video" class="header-link block pr-0.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#diffusers.pipelines.cosmos.pipeline_cosmos3_omni.Cosmos3OmniPipelineOutput.video"><span><svg class="text-smd" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!----><strong>video</strong> — La vidéo générée. Le type exact dépend de <code>output_type</code> | |
| passé au pipeline : une liste de frames PIL pour <code>"pil"</code> (par défaut), un <code>np.ndarray</code> de forme <code>[T, H, W, C]</code> pour <code>"np"</code>, un <code>torch.Tensor</code> de forme <code>[T, C, H, W]</code> pour <code>"pt"</code>, ou un tenseur latent brut | |
| lorsque <code>output_type="latent"</code>.<!----></span></span></li><li class="text-base !pl-4 my-3 rounded "><span class="group flex space-x-1.5 items-start"><a id="diffusers.pipelines.cosmos.pipeline_cosmos3_omni.Cosmos3OmniPipelineOutput.sound" class="header-link block pr-0.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#diffusers.pipelines.cosmos.pipeline_cosmos3_omni.Cosmos3OmniPipelineOutput.sound"><span><svg class="text-smd" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!----><strong>sound</strong> — Forme d'onde audio décodée de forme <code>[C, N]</code>. <code>None</code> lorsque | |
| <code>enable_sound=False</code>.<!----></span></span></li><li class="text-base !pl-4 my-3 rounded "><span class="group flex space-x-1.5 items-start"><a id="diffusers.pipelines.cosmos.pipeline_cosmos3_omni.Cosmos3OmniPipelineOutput.action" class="header-link block pr-0.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#diffusers.pipelines.cosmos.pipeline_cosmos3_omni.Cosmos3OmniPipelineOutput.action"><span><svg class="text-smd" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!----><strong>action</strong> — Jetons d'action prédits. <code>None</code> sauf si un mode d'action prédit des actions.<!----></span></span></li><!--]--></ul><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--></div></div><!----> <p>Classe de données de sortie pour <a href="/docs/diffusers/main/fr/api/pipelines/cosmos3#diffusers.Cosmos3OmniPipeline">Cosmos3OmniPipeline</a>.</p></div> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/diffusers/blob/main/docs/source/en/api/pipelines/cosmos3.md" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg><!----> <span><span class="underline">Update</span> on GitHub</span></a><!----> <p></p><!--]--><!----><!--]--><!--]--><!--]--> <!--[-1--><!--]--><!--]--> | |
| <script> | |
| { | |
| __sveltekit_gxjg5b = { | |
| base: "/docs/diffusers/main/en", | |
| assets: "/docs/diffusers/main/en" | |
| }; | |
| const element = document.currentScript.parentElement; | |
| Promise.all([ | |
| import("/docs/diffusers/main/en/_app/immutable/entry/start.U6TUchwJ.js"), | |
| import("/docs/diffusers/main/en/_app/immutable/entry/app.3qKhPoX-.js") | |
| ]).then(([kit, app]) => { | |
| kit.start(app, element, { | |
| node_ids: [0, 7], | |
| data: [null,null], | |
| form: null, | |
| error: null | |
| }); | |
| }); | |
| } | |
| </script> | |
Xet Storage Details
- Size:
- 235 kB
- Xet hash:
- 61d176cbcaecad8a762b454493cc7a96a98aae8b78da6ed0009e4d3f059d5d01
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.