Buckets:
| <meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"Cosmos 3","local":"cosmos-3","sections":[{"title":"Những điểm mới trong Cosmos 3","local":"whats-new-in-cosmos-3","sections":[],"depth":2},{"title":"Các checkpoint khả dụng","local":"available-checkpoints","sections":[],"depth":2},{"title":"Giảm nhiễu FP8 hỗn hợp W8A8/W8A16","local":"fp8-mixed-w8a8w8a16-denoising","sections":[],"depth":2},{"title":"Mẫu hóa lại prompt","local":"prompt-upsampling","sections":[],"depth":2},{"title":"Văn bản thành video","local":"text-to-video","sections":[],"depth":2},{"title":"Văn bản sang hình ảnh","local":"text-to-image","sections":[],"depth":2},{"title":"Hình ảnh sang video","local":"image-to-video","sections":[],"depth":2},{"title":"Video-to-video","local":"video-to-video","sections":[],"depth":2},{"title":"Video-to-video có âm thanh","local":"video-to-video-with-sound","sections":[],"depth":2},{"title":"Tạo video từ văn bản có âm thanh","local":"text-to-video-with-sound","sections":[],"depth":2},{"title":"Tạo có điều kiện theo hành động","local":"action-conditioned-generation","sections":[{"title":"Chính sách hành động","local":"action-policy","sections":[],"depth":3}],"depth":2},{"title":"SeaCache","local":"seacache","sections":[],"depth":2},{"title":"Song song ngữ cảnh","local":"context-parallelism","sections":[{"title":"Chạy","local":"run-it","sections":[],"depth":3},{"title":"Đào tạo các mô hình lớn với song song tensor","local":"fitting-large-models-with-tensor-parallelism","sections":[],"depth":3},{"title":"Sử dụng trong pipeline mô-đun của riêng bạn","local":"use-it-in-your-own-modular-pipeline","sections":[],"depth":3}],"depth":2},{"title":"Các mẫu metadata","local":"metadata-templates","sections":[],"depth":2},{"title":"Trình kiểm tra an toàn","local":"safety-checker","sections":[],"depth":2},{"title":"Cosmos3OmniPipeline","local":"diffusers.Cosmos3OmniPipeline","sections":[],"depth":2},{"title":"Cosmos3OmniModularPipeline","local":"cosmos3omnimodularpipeline","sections":[{"title":"Các ví dụ mô-đun cho tất cả các quy trình làm việc hiện có","local":"modular-examples-for-all-existing-workflows","sections":[],"depth":3},{"title":"Chuyển giao mô-đun (kiểm soát cấu trúc)","local":"modular-transfer-structural-control","sections":[],"depth":3},{"title":"Chuyển văn bản thành ảnh và ảnh thành video đã được chưng cất (ít bước)","local":"diffusers.Cosmos3OmniModularPipeline","sections":[],"depth":3}],"depth":2},{"title":"Cosmos3DistilledModularPipeline","local":"diffusers.Cosmos3DistilledModularPipeline","sections":[],"depth":2},{"title":"CosmosActionCondition","local":"diffusers.CosmosActionCondition","sections":[],"depth":2},{"title":"Cosmos3OmniPipelineOutput","local":"diffusers.pipelines.cosmos.pipeline_cosmos3_omni.Cosmos3OmniPipelineOutput","sections":[],"depth":2}],"depth":1}"/> | |
| <link href="/docs/diffusers/main/en/_app/immutable/entry/start.U6TUchwJ.js" rel="modulepreload"> | |
| <link href="/docs/diffusers/main/en/_app/immutable/chunks/yxSHiGDF.js" rel="modulepreload"> | |
| <link href="/docs/diffusers/main/en/_app/immutable/chunks/CPfOwUO_.js" rel="modulepreload"> | |
| <link href="/docs/diffusers/main/en/_app/immutable/entry/app.3qKhPoX-.js" rel="modulepreload"> | |
| <link href="/docs/diffusers/main/en/_app/immutable/chunks/BYe-f74j.js" rel="modulepreload"> | |
| <link href="/docs/diffusers/main/en/_app/immutable/chunks/DnVIJfC5.js" rel="modulepreload"> | |
| <link href="/docs/diffusers/main/en/_app/immutable/chunks/DsnmJJEf.js" rel="modulepreload"> | |
| <link href="/docs/diffusers/main/en/_app/immutable/chunks/D32hXVzh.js" rel="modulepreload"> | |
| <link href="/docs/diffusers/main/en/_app/immutable/nodes/0.DqpuBF3n.js" rel="modulepreload"> | |
| <link href="/docs/diffusers/main/en/_app/immutable/chunks/Dyp9xKq8.js" rel="modulepreload"> | |
| <link href="/docs/diffusers/main/en/_app/immutable/nodes/7.MVmUqGUj.js" rel="modulepreload"> | |
| <link href="/docs/diffusers/main/en/_app/immutable/chunks/Dk44y1gU.js" rel="modulepreload"> | |
| <link href="/docs/diffusers/main/en/_app/immutable/chunks/CYniLWKs.js" rel="modulepreload"> | |
| <!--if5dit--><meta name="hf:doc:metadata" content="{"title":"Cosmos 3","local":"cosmos-3","sections":[{"title":"What’s new in Cosmos 3","local":"whats-new-in-cosmos-3","sections":[],"depth":2},{"title":"Available checkpoints","local":"available-checkpoints","sections":[],"depth":2},{"title":"FP8 mixed W8A8/W8A16 denoising","local":"fp8-mixed-w8a8w8a16-denoising","sections":[],"depth":2},{"title":"Prompt upsampling","local":"prompt-upsampling","sections":[],"depth":2},{"title":"Text-to-video","local":"text-to-video","sections":[],"depth":2},{"title":"Text-to-image","local":"text-to-image","sections":[],"depth":2},{"title":"Image-to-video","local":"image-to-video","sections":[],"depth":2},{"title":"Video-to-video","local":"video-to-video","sections":[],"depth":2},{"title":"Video-to-video with sound","local":"video-to-video-with-sound","sections":[],"depth":2},{"title":"Text-to-video with sound","local":"text-to-video-with-sound","sections":[],"depth":2},{"title":"Action-conditioned generation","local":"action-conditioned-generation","sections":[{"title":"Action policy","local":"action-policy","sections":[],"depth":3}],"depth":2},{"title":"SeaCache","local":"seacache","sections":[],"depth":2},{"title":"Context parallelism","local":"context-parallelism","sections":[{"title":"Run it","local":"run-it","sections":[],"depth":3},{"title":"Fitting large models with tensor parallelism","local":"fitting-large-models-with-tensor-parallelism","sections":[],"depth":3},{"title":"Use it in your own modular pipeline","local":"use-it-in-your-own-modular-pipeline","sections":[],"depth":3}],"depth":2},{"title":"Metadata templates","local":"metadata-templates","sections":[],"depth":2},{"title":"Safety checker","local":"safety-checker","sections":[],"depth":2},{"title":"Cosmos3OmniPipeline","local":"diffusers.Cosmos3OmniPipeline","sections":[],"depth":2},{"title":"Cosmos3OmniModularPipeline","local":"cosmos3omnimodularpipeline","sections":[{"title":"Modular examples for all existing workflows","local":"modular-examples-for-all-existing-workflows","sections":[],"depth":3},{"title":"Modular transfer (structural control)","local":"modular-transfer-structural-control","sections":[],"depth":3},{"title":"Distilled (few-step) text-to-image and image-to-video","local":"diffusers.Cosmos3OmniModularPipeline","sections":[],"depth":3}],"depth":2},{"title":"Cosmos3DistilledModularPipeline","local":"diffusers.Cosmos3DistilledModularPipeline","sections":[],"depth":2},{"title":"CosmosActionCondition","local":"diffusers.CosmosActionCondition","sections":[],"depth":2},{"title":"Cosmos3OmniPipelineOutput","local":"diffusers.pipelines.cosmos.pipeline_cosmos3_omni.Cosmos3OmniPipelineOutput","sections":[],"depth":2}],"depth":1}"/><!----> | |
| <link href="/docs/diffusers/main/en/_app/immutable/assets/0.tn0RQdqM.css" rel="modulepreload"> <!--[--><!--[0--><!--[--><!--[0--><!--[--><p></p> <!--[0--><h1 class="relative group"><a id="cosmos-3" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#cosmos-3"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Cosmos 3<!----></span></h1><!--]--><!----> <p>NVIDIA Cosmos 3 là một mô hình nền tảng thế giới (WFM) thống nhất cho Physical AI — một mô hình omni duy nhất kết hợp tạo thế giới, suy luận vật lý và tạo hành động. Nó thay thế các mô hình Predict, Reason và Transfer riêng biệt từ các bản phát hành Cosmos trước đó: dù bạn đang xây dựng cho robot, xe tự hành hay không gian thông minh, Cosmos 3 cung cấp cho bạn một nền tảng duy nhất để mô phỏng và hiểu thế giới vật lý.</p> <p>Những gì được phát hành cùng bản cập nhật này:</p> <ul><li>Các mô hình trên Hugging Face Hub có thẻ mô hình và giấy phép</li> <li>Tích hợp Cosmos 3 Diffusers cho các pipeline tạo (trang này)</li> <li>Post-training scripts để tinh chỉnh Cosmos 3 trên dữ liệu của bạn</li> <li>Các bộ dữ liệu tổng hợp (SDG) mở cho AI vật lý</li></ul> <!--[1--><h2 class="relative group"><a id="whats-new-in-cosmos-3" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#whats-new-in-cosmos-3"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Những điểm mới trong Cosmos 3<!----></span></h2><!--]--><!----> <p>Thay đổi lớn nhất so với các phiên bản Cosmos trước đó là Cosmos 3 là một <em>mô hình omni</em>, được xây dựng trên kiến trúc Mixture-of-Transformers (MoT). Trước đây, các nhà phát triển làm việc với các mô hình riêng biệt cho việc tạo thế giới (Predict), tạo có kiểm soát (Transfer), hiểu cảnh (Reason) và tạo chính sách hành động. Cosmos 3 thống nhất tất cả những điều này trong một mô hình duy nhất, có khả năng suy luận và tạo ra nội dung xuyên suốt các phương thức (modalities) trong một lần truyền tiến (forward pass) duy nhất.</p> <p>Từ một mô hình, bạn có thể:</p> <ul><li>Tạo ra các thế giới video có tính khả thi về mặt vật lý từ văn bản, hình ảnh hoặc đầu vào hành động (tạo video từ hình ảnh, tạo video từ văn bản, tạo video có điều kiện theo hành động).</li> <li>Suy luận về các thuộc tính vật lý như chuyển động, quan hệ nhân quả và quan hệ không gian.</li> <li>Dự đoán các chuỗi video và hành động trong tương lai từ trạng thái hiện tại.</li> <li>Chuyển đổi cảnh giữa các góc nhìn và điều kiện khác nhau với kiểm soát cấu trúc <em>(sắp ra mắt)</em>.</li></ul> <p>Bên trong, một <code>Cosmos3OmniTransformer</code> duy nhất chạy mô hình ngôn ngữ kiểu Qwen song song với đường dẫn sinh bằng diffusion: các token văn bản đi qua luồng “hiểu” có tính nhân quả trong khi các latent video và âm thanh đi qua luồng “sinh” có chú ý hai chiều, được kết nối bằng 3D multimodal RoPE. Xem <a href="https://huggingface.co/papers/2501.03575" rel="nofollow">bài báo về Nền tảng Mô hình Nền tảng Thế giới Cosmos</a> để tìm hiểu bối cảnh kiến trúc.</p> <!--[1--><h2 class="relative group"><a id="available-checkpoints" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#available-checkpoints"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Các checkpoint khả dụng<!----></span></h2><!--]--><!----> <p>Hai checkpoint được phát hành trên Hub — <a href="https://huggingface.co/nvidia/Cosmos3-Nano" rel="nofollow"><code>nvidia/Cosmos3-Nano</code></a> (nhỏ hơn, nhanh hơn) và <a href="https://huggingface.co/nvidia/Cosmos3-Super" rel="nofollow"><code>nvidia/Cosmos3-Super</code></a> (lớn hơn, chất lượng cao hơn). Cùng một lớp pipeline hỗ trợ text-to-image, text-to-video, image-to-video và (với checkpoint có khả năng tạo âm thanh) text+image-to-video-with-sound — hãy chọn một repo và sử dụng tab riêng cho từng model trong mỗi workflow bên dưới.</p> <blockquote class="tip"><p>Hãy chắc chắn xem qua <a href="../../using-diffusers/schedulers">hướng dẫn</a> về Schedulers để tìm hiểu cách khám phá sự đánh đổi giữa tốc độ và chất lượng của scheduler, và xem phần <a href="../../using-diffusers/loading#reusing-models-in-multiple-pipelines">tái sử dụng các thành phần giữa các pipeline</a> để tìm hiểu cách nạp các thành phần giống nhau vào nhiều pipeline một cách hiệu quả.</p></blockquote> <!--[1--><h2 class="relative group"><a id="fp8-mixed-w8a8w8a16-denoising" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#fp8-mixed-w8a8w8a16-denoising"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Giảm nhiễu FP8 hỗn hợp W8A8/W8A16<!----></span></h2><!--]--><!----> <p>Các checkpoint FP8 chính thức của ModelOpt cho Nano, Super và Edge được phát hành dưới dạng các repo Hub riêng: <a href="https://huggingface.co/nvidia/Cosmos3-Nano-FP8" rel="nofollow"><code>nvidia/Cosmos3-Nano-FP8</code></a>, <a href="https://huggingface.co/nvidia/Cosmos3-Super-FP8" rel="nofollow"><code>nvidia/Cosmos3-Super-FP8</code></a> và <a href="https://huggingface.co/nvidia/Cosmos3-Edge-FP8" rel="nofollow"><code>nvidia/Cosmos3-Edge-FP8</code></a>. Hãy nạp <code>main</code> của repo FP8. <a href="https://huggingface.co/nvidia/Cosmos3-Super-Image2Video" rel="nofollow"><code>nvidia/Cosmos3-Super-Image2Video</code></a> không có repo FP8 riêng; hãy nạp nó với <code>revision="fp8"</code>.</p> <p>Tất cả các checkpoint này đều được lượng tử hóa theo cùng một cách. <strong>W8A8</strong> sử dụng trọng số 8-bit và kích hoạt 8-bit (GEMM ModelOpt được khôi phục). <strong>W8A16</strong> tái sử dụng chính những trọng số 8-bit đó nhưng bỏ qua lượng tử hóa kích hoạt: trọng số FP8 được khử lượng tử hóa và một phép toán tuyến tính tiêu chuẩn được thực hiện trên các kích hoạt BF16/FP16/FP32.</p> <p>Việc chạy W8A8 ở mỗi bước có thể gây ra hiện tượng nhấp nháy rõ rệt trong quá trình tạo video nhiều bước. Do đó, các checkpoint video Nano, Super, Super-I2V và Edge FP8 khai báo một lịch trình trong <code>transformer/config.json</code>: <strong>W8A16 ở 3 bước đầu và 3 bước cuối</strong>, <strong>W8A8 ở phần giữa</strong>. Diffusers đọc các giá trị đếm này từ checkpoint thay vì hardcode. Độ chính xác được chọn một lần cho mỗi bước của scheduler để các cuộc gọi cond/uncond của classifier-free guidance khớp nhau.</p> <p>Việc tạo ảnh và các checkpoint đã chưng cất theo kiểu ít bước không cho thấy hiện tượng nhấp nháy, do đó Super-T2I và các kho lưu trữ chưng cất 4 bước FP8 không khai báo lịch trình và duy trì W8A8 ở mọi bước. Lịch trình này cũng chỉ dành cho <strong>ModelOpt FP8</strong>: các backend lượng tử hóa khác (ví dụ TorchAO) vẫn giữ nguyên các forward gốc của chúng.</p> <p>Tải một kho FP8 có cùng đường dẫn khôi phục như <a href="../../quantization/modelopt">hướng dẫn ModelOpt</a>. Kho đã chứa cấu hình lượng tử hóa. Độ chính xác hỗn hợp sau đó sẽ tự động theo checkpoint:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Đã sao chép</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniPipeline | |
| pipe = Cosmos3OmniPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Nano-FP8"</span>, | |
| dtype=torch.bfloat16, | |
| device_map=<span class="hljs-string">"cuda"</span>, | |
| ) | |
| result = pipe(prompt=<span class="hljs-string">"..."</span>, num_inference_steps=<span class="hljs-number">35</span>)<!----></pre></div><!----> <p>Hai lựa chọn khi tạo:</p> <ul><li><strong>Mặc định</strong> (<code>mixed_precision_format=None</code>): nếu checkpoint khai báo <code>diffusion_step_policy</code>, chạy W8A16 ở N bước đầu/cuối và W8A8 bản địa ở phần giữa. Đó là công thức dự kiến cho <strong>video</strong> FP8 nhiều bước (ít nhấp nháy hơn so với toàn bộ W8A8). FP8 4 bước đã chưng cất và Super-T2I FP8 bỏ qua chính sách, nên mặc định đã là toàn bộ W8A8.</li> <li><strong><code>mixed_precision_format="none"</code></strong>: giữ mọi bước ở chế độ W8A8 gốc. Nhanh hơn, vì W8A16 là dequant + <code>torch.nn.functional.linear</code> thay vì GEMM FP8 được khôi phục, nhưng video nhiều bước có thể bị nhấp nháy. Dùng chế độ này để A/B lịch trình hoặc để khớp với baseline lượng tử hóa hoàn toàn.</li></ul> <p>Trên một workstation Blackwell, Cosmos3-Nano-FP8 ở 720×1280 / 35 bước chậm hơn khoảng <strong>27%</strong> (T2I) và <strong>13%</strong> (T2V 49 khung hình) khi dùng lịch trình hỗn hợp mặc định so với <code>"none"</code>. Những con số này không phải là cam kết về thông lượng. Chỉ truyền <code>"fp8"</code> để ép lịch trình first/last-N trên checkpoint ModelOpt FP8 không có policy.</p> <!--[1--><h2 class="relative group"><a id="prompt-upsampling" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#prompt-upsampling"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Mẫu hóa lại prompt<!----></span></h2><!--]--><!----> <p>Cosmos 3 được huấn luyện trên các chú thích dài, mô tả chi tiết. Để đạt chất lượng tối ưu, các prompt văn bản ngắn nên được <strong>mở rộng (upsample) thành một cấu trúc JSON cụ thể</strong> trước khi được truyền vào pipeline. Công cụ mở rộng (upsampler) nằm trong gói <a href="https://github.com/NVIDIA/cosmos-framework" rel="nofollow">cosmos-framework</a>.</p> <p>Bắt đầu từ một prompt văn bản thuần túy ngắn gọn và lưu nó vào <code>assets/prompt.txt</code>. Đối với ví dụ text-to-video bên dưới, prompt gốc là <em>“A robotic arm is cleaning a plate in a kitchen”</em>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Đã sao chép</div><!----></button><!----></div> <pre class="language-bash "><!----><span class="hljs-built_in">mkdir</span> -p assets | |
| <span class="hljs-built_in">echo</span> <span class="hljs-string">"A robotic arm is cleaning a plate in a kitchen"</span> > assets/prompt.txt<!----></pre></div><!----> <p>Sau đó, cài đặt framework và chạy upsampler. Ví dụ bên dưới thực hiện upsampling cho text-to-video bằng Opus-4.6:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Đã sao chép</div><!----></button><!----></div> <pre class="language-bash "><!---->git <span class="hljs-built_in">clone</span> https://github.com/NVIDIA/cosmos-framework.git packages/cosmos-framework | |
| pip install -e packages/cosmos-framework | |
| <span class="hljs-built_in">export</span> PROMPT_UPSAMPLER_ENDPOINT_URL=<span class="hljs-string">"https://api.anthropic.com/v1/"</span> | |
| <span class="hljs-built_in">export</span> PROMPT_UPSAMPLER_MODEL_NAME=<span class="hljs-string">"claude-opus-4-6"</span> | |
| <span class="hljs-built_in">export</span> PROMPT_UPSAMPLER_API_TOKEN=<span class="hljs-string">"<your_token>"</span> | |
| python -m cosmos_framework.inference.prompt_upsampling \ | |
| --input assets/prompt.txt \ | |
| --output assets/example_t2v_prompt.json \ | |
| --mode text2video \ | |
| --endpoint-url <span class="hljs-string">"<span class="hljs-variable">${PROMPT_UPSAMPLER_ENDPOINT_URL}</span>"</span> \ | |
| --model <span class="hljs-string">"<span class="hljs-variable">${PROMPT_UPSAMPLER_MODEL_NAME}</span>"</span> \ | |
| --api-token <span class="hljs-string">"<span class="hljs-variable">${PROMPT_UPSAMPLER_API_TOKEN}</span>"</span> \ | |
| --resolution 720 \ | |
| --aspect-ratio <span class="hljs-string">"16,9"</span><!----></pre></div><!----> <p>Chuyển <code>--mode</code> để khớp với quy trình làm việc bạn đang nhắm tới (<code>text2image</code>, <code>text2video</code>, <code>image2video</code>). Lệnh sẽ ghi các prompt đã được upsample vào tệp <code>--output</code> dưới dạng mảng JSON (một đối tượng cho mỗi dòng không rỗng trong <code>--input</code>); thay vào đó, hãy truyền một đường dẫn <code>.jsonl</code> để nhận một đối tượng JSON cho mỗi dòng. Đối với <code>image2video</code>, bạn cũng phải cung cấp ảnh điều kiện thông qua <code>--image-url</code> (một URL hoặc đường dẫn cục bộ) hoặc <code>--image-list</code> (một ảnh cho mỗi prompt).</p> <p>Một prompt tích cực đã được pre-upsample (<code>assets/example_t2v_prompt.json</code>) và một prompt tiêu cực (<code>assets/negative_prompt.json</code>) được cung cấp để tiện lợi, và được sử dụng trong các ví dụ tạo bên dưới. Các ví dụ này nạp các tệp JSON đó và truyền chúng vào pipeline dưới dạng chuỗi JSON thông qua <code>json.dumps(...)</code>.</p> <!--[1--><h2 class="relative group"><a id="text-to-video" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#text-to-video"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Văn bản thành video<!----></span></h2><!--]--><!----> <p>Tạo nhiều khung hình dựa trên văn bản. Chọn <code>num_frames</code> dựa trên thời lượng mục tiêu — mặc định <code>num_frames=189</code> tạo ra ≈ 7,9 giây ở 24 FPS. Prompt và negative prompt được đọc từ các tệp đã được JSON-upsampled như mô tả trong <a href="#prompt-upsampling">Prompt upsampling</a>.</p> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"><!--[--><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd border-gray-800 bg-black dark:bg-gray-700 text-white">Nano</div><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd text-gray-500 cursor-pointer opacity-90 hover:text-gray-700 dark:hover:text-gray-200 hover:shadow-sm">Tuyệt vời</div><!--]--></div> <div class="language-select"><div><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Đã sao chép</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniPipeline | |
| <span class="hljs-keyword">from</span> diffusers.schedulers.scheduling_unipc_multistep <span class="hljs-keyword">import</span> UniPCMultistepScheduler | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video | |
| <span class="hljs-comment"># JSON-upsampled positive and negative prompts (see "Prompt upsampling" above).</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_t2v_prompt.json"</span>)) | |
| negative_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/negative_prompt.json"</span>)) | |
| pipe = Cosmos3OmniPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Nano"</span>, dtype=torch.bfloat16, device_map=<span class="hljs-string">"cuda"</span> <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| ) | |
| pipe.scheduler = UniPCMultistepScheduler.from_config( | |
| pipe.scheduler.config, flow_shift=<span class="hljs-number">10.0</span>, use_karras_sigmas=<span class="hljs-literal">False</span> | |
| ) | |
| result = pipe( | |
| prompt=json.dumps(json_prompt), | |
| negative_prompt=json.dumps(negative_prompt), | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| num_inference_steps=<span class="hljs-number">35</span>, | |
| guidance_scale=<span class="hljs-number">6.0</span>, | |
| fps=<span class="hljs-number">24.0</span>, | |
| ) | |
| <span class="hljs-comment"># macro_block_size=1 allows arbitrary frame sizes (Cosmos3 outputs are not always divisible by 16).</span> | |
| export_to_video(result.video, <span class="hljs-string">"cosmos3_t2v.mp4"</span>, fps=<span class="hljs-number">24</span>, macro_block_size=<span class="hljs-number">1</span>)<!----></pre></div><!----></div><!----> <div hidden=""><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Đã sao chép</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniPipeline | |
| <span class="hljs-keyword">from</span> diffusers.schedulers.scheduling_unipc_multistep <span class="hljs-keyword">import</span> UniPCMultistepScheduler | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video | |
| <span class="hljs-comment"># JSON-upsampled positive and negative prompts (see "Prompt upsampling" above).</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_t2v_prompt.json"</span>)) | |
| negative_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/negative_prompt.json"</span>)) | |
| pipe = Cosmos3OmniPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Super"</span>, dtype=torch.bfloat16, device_map=<span class="hljs-string">"cuda"</span> <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| ) | |
| pipe.scheduler = UniPCMultistepScheduler.from_config( | |
| pipe.scheduler.config, flow_shift=<span class="hljs-number">10.0</span>, use_karras_sigmas=<span class="hljs-literal">False</span> | |
| ) | |
| result = pipe( | |
| prompt=json.dumps(json_prompt), | |
| negative_prompt=json.dumps(negative_prompt), | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| num_inference_steps=<span class="hljs-number">35</span>, | |
| guidance_scale=<span class="hljs-number">6.0</span>, | |
| fps=<span class="hljs-number">24.0</span>, | |
| ) | |
| <span class="hljs-comment"># macro_block_size=1 allows arbitrary frame sizes (Cosmos3 outputs are not always divisible by 16).</span> | |
| export_to_video(result.video, <span class="hljs-string">"cosmos3_t2v.mp4"</span>, fps=<span class="hljs-number">24</span>, macro_block_size=<span class="hljs-number">1</span>)<!----></pre></div><!----></div><!----><!----></div><!----> <!--[1--><h2 class="relative group"><a id="text-to-image" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#text-to-image"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Văn bản sang hình ảnh<!----></span></h2><!--]--><!----> <p>Tạo khung đơn. Mô hình chỉ được điều kiện bởi prompt văn bản; truyền <code>num_frames=1</code>. Upsample với <code>--mode text2image</code> để tạo prompt JSON.</p> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"><!--[--><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd border-gray-800 bg-black dark:bg-gray-700 text-white">Nano</div><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd text-gray-500 cursor-pointer opacity-90 hover:text-gray-700 dark:hover:text-gray-200 hover:shadow-sm">Tuyệt vời</div><!--]--></div> <div class="language-select"><div><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Đã sao chép</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniPipeline | |
| <span class="hljs-comment"># JSON-upsampled prompt (see "Prompt upsampling" above).</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_t2i_prompt.json"</span>)) | |
| pipe = Cosmos3OmniPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Nano"</span>, dtype=torch.bfloat16, device_map=<span class="hljs-string">"cuda"</span> <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| ) | |
| result = pipe(prompt=json.dumps(json_prompt), num_frames=<span class="hljs-number">1</span>, height=<span class="hljs-number">720</span>, width=<span class="hljs-number">1280</span>) | |
| result.video[<span class="hljs-number">0</span>].save(<span class="hljs-string">"cosmos3_t2i.jpg"</span>, <span class="hljs-built_in">format</span>=<span class="hljs-string">"JPEG"</span>, quality=<span class="hljs-number">85</span>)<!----></pre></div><!----></div><!----> <div hidden=""><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Đã sao chép</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniPipeline | |
| <span class="hljs-comment"># JSON-upsampled prompt (see "Prompt upsampling" above).</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_t2i_prompt.json"</span>)) | |
| pipe = Cosmos3OmniPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Super"</span>, dtype=torch.bfloat16, device_map=<span class="hljs-string">"cuda"</span> <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| ) | |
| result = pipe(prompt=json.dumps(json_prompt), num_frames=<span class="hljs-number">1</span>, height=<span class="hljs-number">720</span>, width=<span class="hljs-number">1280</span>) | |
| result.video[<span class="hljs-number">0</span>].save(<span class="hljs-string">"cosmos3_t2i.jpg"</span>, <span class="hljs-built_in">format</span>=<span class="hljs-string">"JPEG"</span>, quality=<span class="hljs-number">85</span>)<!----></pre></div><!----></div><!----><!----></div><!----> <!--[1--><h2 class="relative group"><a id="image-to-video" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#image-to-video"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Hình ảnh sang video<!----></span></h2><!--]--><!----> <p>Truyền ảnh điều kiện qua <code>image=</code>. Pipeline sẽ neo khung 0 vào ảnh được cung cấp và khử nhiễu các khung còn lại. Ảnh sẽ được thay đổi kích thước trong khi giữ nguyên tỷ lệ khung hình, cắt theo tâm để khớp với kích thước đầu ra yêu cầu, và được chuẩn hóa với làm tròn tương đương uint8. Dùng <code>--mode image2video</code> để lấy mẫu lên và tạo ra prompt JSON.</p> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"><!--[--><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd border-gray-800 bg-black dark:bg-gray-700 text-white">Nano</div><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd text-gray-500 cursor-pointer opacity-90 hover:text-gray-700 dark:hover:text-gray-200 hover:shadow-sm">Tuyệt vời</div><!--]--></div> <div class="language-select"><div><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Đã sao chép</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniPipeline | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video, load_image | |
| <span class="hljs-comment"># JSON-upsampled positive and negative prompts (see "Prompt upsampling" above).</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_i2v_prompt.json"</span>)) | |
| negative_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/negative_prompt_i2v.json"</span>)) | |
| pipe = Cosmos3OmniPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Nano"</span>, dtype=torch.bfloat16, device_map=<span class="hljs-string">"cuda"</span> <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| ) | |
| image = load_image( | |
| <span class="hljs-string">"https://github.com/nvidia-cosmos/cosmos-dependencies/releases/download/assets/robot_153.jpg"</span> | |
| ) | |
| result = pipe( | |
| prompt=json.dumps(json_prompt), | |
| negative_prompt=json.dumps(negative_prompt), | |
| image=image, | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| fps=<span class="hljs-number">24.0</span>, | |
| ) | |
| <span class="hljs-comment"># macro_block_size=1 allows arbitrary frame sizes (Cosmos3 outputs are not always divisible by 16).</span> | |
| export_to_video(result.video, <span class="hljs-string">"cosmos3_i2v.mp4"</span>, fps=<span class="hljs-number">24</span>, macro_block_size=<span class="hljs-number">1</span>)<!----></pre></div><!----></div><!----> <div hidden=""><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Đã sao chép</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniPipeline | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video, load_image | |
| <span class="hljs-comment"># JSON-upsampled positive and negative prompts (see "Prompt upsampling" above).</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_i2v_prompt.json"</span>)) | |
| negative_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/negative_prompt_i2v.json"</span>)) | |
| pipe = Cosmos3OmniPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Super"</span>, dtype=torch.bfloat16, device_map=<span class="hljs-string">"cuda"</span> <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| ) | |
| image = load_image( | |
| <span class="hljs-string">"https://github.com/nvidia-cosmos/cosmos-dependencies/releases/download/assets/robot_153.jpg"</span> | |
| ) | |
| result = pipe( | |
| prompt=json.dumps(json_prompt), | |
| negative_prompt=json.dumps(negative_prompt), | |
| image=image, | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| fps=<span class="hljs-number">24.0</span>, | |
| ) | |
| <span class="hljs-comment"># macro_block_size=1 allows arbitrary frame sizes (Cosmos3 outputs are not always divisible by 16).</span> | |
| export_to_video(result.video, <span class="hljs-string">"cosmos3_i2v.mp4"</span>, fps=<span class="hljs-number">24</span>, macro_block_size=<span class="hljs-number">1</span>)<!----></pre></div><!----></div><!----><!----></div><!----> <!--[1--><h2 class="relative group"><a id="video-to-video" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#video-to-video"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Video-to-video<!----></span></h2><!--]--><!----> <p>Truyền một đoạn video điều kiện thông qua <code>video=</code> (ví dụ từ <code>load_video</code>). Pipeline sẽ neo các khung latent đầu tiên được chỉ định bởi <code>condition_frame_indexes_vision</code> (mặc định là <code>[0, 1]</code>) vào đoạn video và khử nhiễu phần còn lại. Sử dụng <code>condition_video_keep</code> (<code>"first"</code> hoặc <code>"last"</code>) để chọn đầu nào của đoạn video nguồn dài hơn mà các khung điều kiện sẽ được lấy từ đó. Tương tự như các chế độ khác, prompt nên tuân theo cấu trúc JSON mô tả được nêu trong <a href="#prompt-upsampling">Prompt upsampling</a>.</p> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"><!--[--><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd border-gray-800 bg-black dark:bg-gray-700 text-white">Nano</div><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd text-gray-500 cursor-pointer opacity-90 hover:text-gray-700 dark:hover:text-gray-200 hover:shadow-sm">Tuyệt vời</div><!--]--></div> <div class="language-select"><div><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Đã sao chép</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniPipeline | |
| <span class="hljs-keyword">from</span> diffusers.schedulers.scheduling_unipc_multistep <span class="hljs-keyword">import</span> UniPCMultistepScheduler | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video, load_video | |
| <span class="hljs-comment"># JSON-upsampled positive and negative prompts (see "Prompt upsampling" above).</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_v2v_prompt.json"</span>)) | |
| negative_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/negative_prompt_i2v.json"</span>)) | |
| pipe = Cosmos3OmniPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Nano"</span>, dtype=torch.bfloat16, device_map=<span class="hljs-string">"cuda"</span> <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| ) | |
| pipe.scheduler = UniPCMultistepScheduler.from_config( | |
| pipe.scheduler.config, flow_shift=<span class="hljs-number">10.0</span>, use_karras_sigmas=<span class="hljs-literal">False</span> | |
| ) | |
| video = load_video( | |
| <span class="hljs-string">"https://github.com/nvidia-cosmos/cosmos-dependencies/raw/refs/heads/assets/cosmos3/inputs/vision/robot_pouring.mp4"</span> | |
| ) | |
| result = pipe( | |
| prompt=json.dumps(json_prompt), | |
| negative_prompt=json.dumps(negative_prompt), | |
| video=video, | |
| condition_frame_indexes_vision=[<span class="hljs-number">0</span>, <span class="hljs-number">1</span>], | |
| condition_video_keep=<span class="hljs-string">"first"</span>, | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| num_inference_steps=<span class="hljs-number">35</span>, | |
| guidance_scale=<span class="hljs-number">6.0</span>, | |
| fps=<span class="hljs-number">24.0</span>, | |
| ) | |
| <span class="hljs-comment"># macro_block_size=1 allows arbitrary frame sizes (Cosmos3 outputs are not always divisible by 16).</span> | |
| export_to_video(result.video, <span class="hljs-string">"cosmos3_v2v.mp4"</span>, fps=<span class="hljs-number">24</span>, macro_block_size=<span class="hljs-number">1</span>)<!----></pre></div><!----></div><!----> <div hidden=""><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Đã sao chép</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniPipeline | |
| <span class="hljs-keyword">from</span> diffusers.schedulers.scheduling_unipc_multistep <span class="hljs-keyword">import</span> UniPCMultistepScheduler | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video, load_video | |
| <span class="hljs-comment"># JSON-upsampled positive and negative prompts (see "Prompt upsampling" above).</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_v2v_prompt.json"</span>)) | |
| negative_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/negative_prompt_i2v.json"</span>)) | |
| pipe = Cosmos3OmniPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Super"</span>, dtype=torch.bfloat16, device_map=<span class="hljs-string">"cuda"</span> <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| ) | |
| pipe.scheduler = UniPCMultistepScheduler.from_config( | |
| pipe.scheduler.config, flow_shift=<span class="hljs-number">10.0</span>, use_karras_sigmas=<span class="hljs-literal">False</span> | |
| ) | |
| video = load_video( | |
| <span class="hljs-string">"https://github.com/nvidia-cosmos/cosmos-dependencies/raw/refs/heads/assets/cosmos3/inputs/vision/robot_pouring.mp4"</span> | |
| ) | |
| result = pipe( | |
| prompt=json.dumps(json_prompt), | |
| negative_prompt=json.dumps(negative_prompt), | |
| video=video, | |
| condition_frame_indexes_vision=[<span class="hljs-number">0</span>, <span class="hljs-number">1</span>], | |
| condition_video_keep=<span class="hljs-string">"first"</span>, | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| num_inference_steps=<span class="hljs-number">35</span>, | |
| guidance_scale=<span class="hljs-number">6.0</span>, | |
| fps=<span class="hljs-number">24.0</span>, | |
| ) | |
| <span class="hljs-comment"># macro_block_size=1 allows arbitrary frame sizes (Cosmos3 outputs are not always divisible by 16).</span> | |
| export_to_video(result.video, <span class="hljs-string">"cosmos3_v2v.mp4"</span>, fps=<span class="hljs-number">24</span>, macro_block_size=<span class="hljs-number">1</span>)<!----></pre></div><!----></div><!----><!----></div><!----> <!--[1--><h2 class="relative group"><a id="video-to-video-with-sound" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#video-to-video-with-sound"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Video-to-video có âm thanh<!----></span></h2><!--]--><!----> <p>Khi checkpoint chứa <code>sound_tokenizer</code>, hãy thêm <code>enable_sound=True</code> vào lệnh gọi video-to-video để đồng thời tạo ra một bản nhạc nền đồng bộ. Dạng sóng được trả về cùng với video và có thể được ghép vào tệp MP4 bằng <a href="/docs/diffusers/main/vi/api/utilities#diffusers.utils.encode_video">encode_video()</a>.</p> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"><!--[--><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd border-gray-800 bg-black dark:bg-gray-700 text-white">Nano</div><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd text-gray-500 cursor-pointer opacity-90 hover:text-gray-700 dark:hover:text-gray-200 hover:shadow-sm">Tuyệt vời</div><!--]--></div> <div class="language-select"><div><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Đã sao chép</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniPipeline | |
| <span class="hljs-keyword">from</span> diffusers.schedulers.scheduling_unipc_multistep <span class="hljs-keyword">import</span> UniPCMultistepScheduler | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> encode_video, load_video | |
| <span class="hljs-comment"># JSON-upsampled positive and negative prompts (see "Prompt upsampling" above).</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_v2v_prompt.json"</span>)) | |
| negative_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/negative_prompt_i2v.json"</span>)) | |
| pipe = Cosmos3OmniPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Nano"</span>, dtype=torch.bfloat16, device_map=<span class="hljs-string">"cuda"</span> <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| ) | |
| pipe.scheduler = UniPCMultistepScheduler.from_config( | |
| pipe.scheduler.config, flow_shift=<span class="hljs-number">10.0</span>, use_karras_sigmas=<span class="hljs-literal">False</span> | |
| ) | |
| video = load_video( | |
| <span class="hljs-string">"https://github.com/nvidia-cosmos/cosmos-dependencies/raw/refs/heads/assets/cosmos3/inputs/vision/robot_pouring.mp4"</span> | |
| ) | |
| result = pipe( | |
| prompt=json.dumps(json_prompt), | |
| negative_prompt=json.dumps(negative_prompt), | |
| video=video, | |
| condition_frame_indexes_vision=[<span class="hljs-number">0</span>, <span class="hljs-number">1</span>], | |
| condition_video_keep=<span class="hljs-string">"first"</span>, | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| fps=<span class="hljs-number">24.0</span>, | |
| enable_sound=<span class="hljs-literal">True</span>, | |
| ) | |
| encode_video( | |
| result.video, | |
| fps=<span class="hljs-number">24</span>, | |
| audio=result.sound, | |
| audio_sample_rate=pipe.sound_tokenizer.config.sampling_rate, | |
| output_path=<span class="hljs-string">"cosmos3_v2v_with_sound.mp4"</span>, | |
| )<!----></pre></div><!----></div><!----> <div hidden=""><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Đã sao chép</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniPipeline | |
| <span class="hljs-keyword">from</span> diffusers.schedulers.scheduling_unipc_multistep <span class="hljs-keyword">import</span> UniPCMultistepScheduler | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> encode_video, load_video | |
| <span class="hljs-comment"># JSON-upsampled positive and negative prompts (see "Prompt upsampling" above).</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_v2v_prompt.json"</span>)) | |
| negative_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/negative_prompt_i2v.json"</span>)) | |
| pipe = Cosmos3OmniPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Super"</span>, dtype=torch.bfloat16, device_map=<span class="hljs-string">"cuda"</span> <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| ) | |
| pipe.scheduler = UniPCMultistepScheduler.from_config( | |
| pipe.scheduler.config, flow_shift=<span class="hljs-number">10.0</span>, use_karras_sigmas=<span class="hljs-literal">False</span> | |
| ) | |
| video = load_video( | |
| <span class="hljs-string">"https://github.com/nvidia-cosmos/cosmos-dependencies/raw/refs/heads/assets/cosmos3/inputs/vision/robot_pouring.mp4"</span> | |
| ) | |
| result = pipe( | |
| prompt=json.dumps(json_prompt), | |
| negative_prompt=json.dumps(negative_prompt), | |
| video=video, | |
| condition_frame_indexes_vision=[<span class="hljs-number">0</span>, <span class="hljs-number">1</span>], | |
| condition_video_keep=<span class="hljs-string">"first"</span>, | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| fps=<span class="hljs-number">24.0</span>, | |
| enable_sound=<span class="hljs-literal">True</span>, | |
| ) | |
| encode_video( | |
| result.video, | |
| fps=<span class="hljs-number">24</span>, | |
| audio=result.sound, | |
| audio_sample_rate=pipe.sound_tokenizer.config.sampling_rate, | |
| output_path=<span class="hljs-string">"cosmos3_v2v_with_sound.mp4"</span>, | |
| )<!----></pre></div><!----></div><!----><!----></div><!----> <!--[1--><h2 class="relative group"><a id="text-to-video-with-sound" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#text-to-video-with-sound"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Tạo video từ văn bản có âm thanh<!----></span></h2><!--]--><!----> <p>Khi checkpoint chứa <code>sound_tokenizer</code>, hãy truyền <code>enable_sound=True</code> để đồng thời tạo ra một bản nhạc nền đồng bộ. Dạng sóng được trả về cùng với video và có thể được ghép vào tệp MP4 bằng <a href="/docs/diffusers/main/vi/api/utilities#diffusers.utils.encode_video">encode_video()</a>.</p> <p>Đây là cùng một lệnh gọi như ví dụ text-to-video ở trên, với <code>enable_sound=True</code> được thêm vào:</p> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"><!--[--><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd border-gray-800 bg-black dark:bg-gray-700 text-white">Nano</div><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd text-gray-500 cursor-pointer opacity-90 hover:text-gray-700 dark:hover:text-gray-200 hover:shadow-sm">Tuyệt vời</div><!--]--></div> <div class="language-select"><div><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Đã sao chép</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniPipeline | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> encode_video | |
| <span class="hljs-comment"># JSON-upsampled positive and negative prompts (see "Prompt upsampling" above).</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_t2v_sound_prompt.json"</span>)) | |
| negative_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/negative_prompt.json"</span>)) | |
| pipe = Cosmos3OmniPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Nano"</span>, dtype=torch.bfloat16, device_map=<span class="hljs-string">"cuda"</span> <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| ) | |
| result = pipe( | |
| prompt=json.dumps(json_prompt), | |
| negative_prompt=json.dumps(negative_prompt), | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| fps=<span class="hljs-number">24.0</span>, | |
| enable_sound=<span class="hljs-literal">True</span>, | |
| ) | |
| encode_video( | |
| result.video, | |
| fps=<span class="hljs-number">24</span>, | |
| audio=result.sound, | |
| audio_sample_rate=pipe.sound_tokenizer.config.sampling_rate, | |
| output_path=<span class="hljs-string">"cosmos3_with_sound.mp4"</span>, | |
| )<!----></pre></div><!----></div><!----> <div hidden=""><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Đã sao chép</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniPipeline | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> encode_video | |
| <span class="hljs-comment"># JSON-upsampled positive and negative prompts (see "Prompt upsampling" above).</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_t2v_sound_prompt.json"</span>)) | |
| negative_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/negative_prompt.json"</span>)) | |
| pipe = Cosmos3OmniPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Super"</span>, dtype=torch.bfloat16, device_map=<span class="hljs-string">"cuda"</span> <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| ) | |
| result = pipe( | |
| prompt=json.dumps(json_prompt), | |
| negative_prompt=json.dumps(negative_prompt), | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| fps=<span class="hljs-number">24.0</span>, | |
| enable_sound=<span class="hljs-literal">True</span>, | |
| ) | |
| encode_video( | |
| result.video, | |
| fps=<span class="hljs-number">24</span>, | |
| audio=result.sound, | |
| audio_sample_rate=pipe.sound_tokenizer.config.sampling_rate, | |
| output_path=<span class="hljs-string">"cosmos3_with_sound.mp4"</span>, | |
| )<!----></pre></div><!----></div><!----><!----></div><!----> <!--[1--><h2 class="relative group"><a id="action-conditioned-generation" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#action-conditioned-generation"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Tạo có điều kiện theo hành động<!----></span></h2><!--]--><!----> <p>Action runs nhóm mọi đầu vào cụ thể cho hành động vào một <a href="/docs/diffusers/main/vi/api/pipelines/cosmos3#diffusers.CosmosActionCondition">CosmosActionCondition</a> được truyền qua đối số <code>action</code> thay vì các đối số cấp cao <code>image</code> / <code>video</code> / <code>height</code> / <code>width</code>. Đặt <code>resolution_tier</code> (<code>256</code>/<code>480</code>/<code>704</code>/<code>720</code>) gần với độ phân giải gốc của video đầu vào; nó chọn khung nền điều kiện. Cosmos 3 hỗ trợ ba chế độ hành động — <code>policy</code>, <code>forward_dynamics</code> và <code>inverse_dynamics</code>. <code>policy</code> và <code>forward_dynamics</code> chỉ điều kiện trên khung hình đầu tiên (vì vậy cả <code>image</code> và <code>video</code> đều hoạt động), trong khi <code>inverse_dynamics</code> yêu cầu một <code>video</code>. Video điều kiện cho một lần chạy hành động được đặt trên <code>action.video</code> (hoặc <code>action.image</code>), không phải trên đối số <code>video</code> cấp cao của pipeline.</p> <p>Truyền mô tả tác vụ đơn giản dưới dạng <code>prompt</code> và chọn camera bằng <code>action.view_point</code> (mặc định là <code>"ego_view"</code>; ngoài ra còn có <code>"third_person_view"</code>, <code>"wrist_view"</code>, <code>"concat_view"</code>). Pipeline sẽ chuyển đổi các thông tin này thành chú thích JSON có cấu trúc mà mô hình đã được huấn luyện, do đó prompt cho tác vụ không nên được LLM-upsample.</p> <!--[2--><h3 class="relative group"><a id="action-policy" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#action-policy"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Chính sách hành động<!----></span></h3><!--]--><!----> <p>Việc tạo chính sách hành động dự đoán các token video và hành động trong tương lai từ khung quan sát đầu tiên, prompt văn bản và metadata của miền hành động. Ví dụ bên dưới sử dụng miền robot Bridge và ghi chunk hành động được dự đoán vào JSON trong không gian hành động đã chuẩn hóa theo mô hình.</p> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"><!--[--><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd border-gray-800 bg-black dark:bg-gray-700 text-white">Nano</div><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd text-gray-500 cursor-pointer opacity-90 hover:text-gray-700 dark:hover:text-gray-200 hover:shadow-sm">Tuyệt vời</div><!--]--></div> <div class="language-select"><div><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Đã sao chép</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniPipeline, CosmosActionCondition | |
| <span class="hljs-keyword">from</span> diffusers.schedulers.scheduling_unipc_multistep <span class="hljs-keyword">import</span> UniPCMultistepScheduler | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video, load_video | |
| pipe = Cosmos3OmniPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Nano"</span>, dtype=torch.bfloat16, device_map=<span class="hljs-string">"cuda"</span> <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| ) | |
| pipe.scheduler = UniPCMultistepScheduler.from_config( | |
| pipe.scheduler.config, flow_shift=<span class="hljs-number">10.0</span>, use_karras_sigmas=<span class="hljs-literal">False</span> | |
| ) | |
| prompt = <span class="hljs-string">"Put the pot to the left of the purple item."</span> | |
| video = load_video( | |
| <span class="hljs-string">"https://github.com/nvidia-cosmos/cosmos-dependencies/raw/refs/heads/assets/cosmos3/inputs/action/bridge_20260501_0.mp4"</span> | |
| ) | |
| result = pipe( | |
| prompt=prompt, | |
| action=CosmosActionCondition( | |
| mode=<span class="hljs-string">"policy"</span>, | |
| chunk_size=<span class="hljs-number">16</span>, | |
| domain_name=<span class="hljs-string">"bridge_orig_lerobot"</span>, | |
| resolution_tier=<span class="hljs-number">480</span>, | |
| video=video, | |
| view_point=<span class="hljs-string">"ego_view"</span>, | |
| ), | |
| fps=<span class="hljs-number">5</span>, | |
| num_inference_steps=<span class="hljs-number">30</span>, | |
| guidance_scale=<span class="hljs-number">1.0</span>, | |
| use_system_prompt=<span class="hljs-literal">False</span>, | |
| ) | |
| <span class="hljs-comment"># macro_block_size=1 allows arbitrary frame sizes (Cosmos3 outputs are not always divisible by 16).</span> | |
| export_to_video(result.video, <span class="hljs-string">"sample.mp4"</span>, fps=<span class="hljs-number">5</span>, macro_block_size=<span class="hljs-number">1</span>) | |
| <span class="hljs-keyword">if</span> result.action <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> <span class="hljs-literal">None</span>: | |
| <span class="hljs-keyword">with</span> <span class="hljs-built_in">open</span>(<span class="hljs-string">"sample_action.json"</span>, <span class="hljs-string">"w"</span>) <span class="hljs-keyword">as</span> f: | |
| json.dump(result.action[<span class="hljs-number">0</span>].tolist(), f)<!----></pre></div><!----></div><!----> <div hidden=""><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Đã sao chép</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniPipeline, CosmosActionCondition | |
| <span class="hljs-keyword">from</span> diffusers.schedulers.scheduling_unipc_multistep <span class="hljs-keyword">import</span> UniPCMultistepScheduler | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video, load_video | |
| pipe = Cosmos3OmniPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Super"</span>, dtype=torch.bfloat16, device_map=<span class="hljs-string">"cuda"</span> <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| ) | |
| pipe.scheduler = UniPCMultistepScheduler.from_config( | |
| pipe.scheduler.config, flow_shift=<span class="hljs-number">10.0</span>, use_karras_sigmas=<span class="hljs-literal">False</span> | |
| ) | |
| prompt = <span class="hljs-string">"Put the pot to the left of the purple item."</span> | |
| video = load_video( | |
| <span class="hljs-string">"https://github.com/nvidia-cosmos/cosmos-dependencies/raw/refs/heads/assets/cosmos3/inputs/action/bridge_20260501_0.mp4"</span> | |
| ) | |
| result = pipe( | |
| prompt=prompt, | |
| action=CosmosActionCondition( | |
| mode=<span class="hljs-string">"policy"</span>, | |
| chunk_size=<span class="hljs-number">16</span>, | |
| domain_name=<span class="hljs-string">"bridge_orig_lerobot"</span>, | |
| resolution_tier=<span class="hljs-number">480</span>, | |
| video=video, | |
| view_point=<span class="hljs-string">"ego_view"</span>, | |
| ), | |
| fps=<span class="hljs-number">5</span>, | |
| num_inference_steps=<span class="hljs-number">30</span>, | |
| guidance_scale=<span class="hljs-number">1.0</span>, | |
| use_system_prompt=<span class="hljs-literal">False</span>, | |
| ) | |
| <span class="hljs-comment"># macro_block_size=1 allows arbitrary frame sizes (Cosmos3 outputs are not always divisible by 16).</span> | |
| export_to_video(result.video, <span class="hljs-string">"sample.mp4"</span>, fps=<span class="hljs-number">5</span>, macro_block_size=<span class="hljs-number">1</span>) | |
| <span class="hljs-keyword">if</span> result.action <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> <span class="hljs-literal">None</span>: | |
| <span class="hljs-keyword">with</span> <span class="hljs-built_in">open</span>(<span class="hljs-string">"sample_action.json"</span>, <span class="hljs-string">"w"</span>) <span class="hljs-keyword">as</span> f: | |
| json.dump(result.action[<span class="hljs-number">0</span>].tolist(), f)<!----></pre></div><!----></div><!----><!----></div><!----> <!--[1--><h2 class="relative group"><a id="seacache" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#seacache"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->SeaCache<!----></span></h2><!--]--><!----> <p>SeaCache bị vô hiệu hóa theo mặc định. Cosmos 3 hỗ trợ bật nó một cách rõ ràng bằng <a href="/docs/diffusers/main/vi/api/cache#diffusers.SeaCacheConfig">SeaCacheConfig</a>. SeaCache tái sử dụng các phần dư của transformer khi chỉ báo Spectral-Evolution-Aware thay đổi chậm, từ đó giảm số lần thực thi đầy đủ của transformer. Nó tính toán chỉ báo này từ các latent thị giác thô, bao gồm các khung điều kiện sạch cho việc tạo video từ ảnh. Bật nó trên transformer bằng các callback metadata của scheduler từ pipeline:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Đã sao chép</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniPipeline, SeaCacheConfig | |
| pipe = Cosmos3OmniPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Nano"</span>, dtype=torch.bfloat16, device_map=<span class="hljs-string">"cuda"</span> | |
| ) | |
| pipe.transformer.enable_cache(SeaCacheConfig(threshold=<span class="hljs-number">0.2</span>, max_consecutive_cached=<span class="hljs-number">2</span>))<!----></pre></div><!----> <p>Cùng một API ở cấp độ model hoạt động với <a href="/docs/diffusers/main/vi/api/pipelines/cosmos3#diffusers.Cosmos3OmniPipeline">Cosmos3OmniPipeline</a>, <a href="/docs/diffusers/main/vi/api/pipelines/cosmos3#diffusers.Cosmos3OmniModularPipeline">Cosmos3OmniModularPipeline</a> và <a href="/docs/diffusers/main/vi/api/pipelines/cosmos3#diffusers.Cosmos3DistilledModularPipeline">Cosmos3DistilledModularPipeline</a>. SeaCache là xấp xỉ và có thể làm thay đổi kết quả đầu ra được tạo ra. Hãy tắt nó bằng <code>pipe.transformer.disable_cache()</code> khi bạn cần mỗi bước khử nhiễu thực thi đầy đủ transformer. Trạng thái cache được đặt lại sau mỗi lần gọi pipeline, và các nhánh hướng dẫn có điều kiện và không điều kiện giữ các lịch sử độc lập.</p> <p>Cosmos 3 giữ cổng SeaCache nằm bên ngoài các lớp decoder lặp lại, do đó tương thích với biên dịch theo vùng. | |
| Biên dịch các lớp sau khi bật cache:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Đã sao chép</div><!----></button><!----></div> <pre class="language-python "><!---->pipe.transformer.compile_repeated_blocks(fullgraph=<span class="hljs-literal">True</span>)<!----></pre></div><!----> <p>SeaCache cũng hỗ trợ các tiện ích song song ngữ cảnh Ulysses và song song tensor dựa trên DTensor của Cosmos 3 được mô tả bên dưới. Các quyết định về cache được đồng bộ hóa giữa các rank. Không cam kết về việc biên dịch toàn bộ mô hình và các chiến lược chia mô hình khác.</p> <!--[1--><h2 class="relative group"><a id="context-parallelism" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#context-parallelism"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Song song ngữ cảnh<!----></span></h2><!--]--><!----> <p>Đối với video dài hoặc độ phân giải cao, một lần truyền tiến (forward pass) có thể vượt quá ngân sách bộ nhớ và độ trễ của một GPU. Cosmos 3 hỗ trợ <strong>song song ngữ cảnh (context parallelism - CP)</strong> để chia nhỏ chiều chuỗi (sequence dimension) trên nhiều GPU, tách rời phép tính chú ý (attention) để mỗi thiết bị chỉ giữ một phần các token.</p> <p>Cosmos 3 hỗ trợ song song ngữ cảnh <strong>Ulysses</strong> (trao đổi chuỗi/đầu all-to-all). Ring attention không được hỗ trợ.</p> <p>Khác với hầu hết các mô hình diffusers, Cosmos 3 không <strong>gắn</strong> CP vào transformer hay đường dẫn khai báo <code>enable_parallelism()</code>: cơ chế grouped-query attention, các luồng understanding/generation riêng biệt (luồng generation chú ý đến cả hai) và độ dài không đều theo từng luồng không thể được biểu diễn dưới dạng <code>_cp_plan</code>. Thay vào đó, mô hình phơi bày các điểm nối shard/gather dạng no-op nhỏ gọn, và phần cài đặt nằm trong <a href="https://github.com/huggingface/diffusers/blob/main/examples/cosmos3/cosmos_parallel.py" rel="nofollow"><code>examples/cosmos3/cosmos_parallel.py</code></a> — một module tự chứa mà bạn có thể đọc từ đầu đến cuối và tùy biến. Nó cung cấp hai trục sharding trực giao, có thể kết hợp được:</p> <table><thead><tr><th>Trợ lý</th><th>Các phân mảnh</th><th>Dùng cho</th></tr></thead><tbody><tr><td><code>enable_cosmos3_context_parallel(transformer, cp_mesh)</code></td><td>chuỗi (CP / Ulysses)</td><td>độ trễ trên một model vừa với một GPU (<code>Nano</code>)</td></tr><tr><td><code>enable_cosmos3_tensor_parallel(transformer, tp_mesh)</code></td><td>weights (TP)</td><td>đào tạo một mô hình không vừa với một GPU (<code>Super</code>)</td></tr></tbody></table> <p>Sử dụng một trong hai hoặc cả hai cùng nhau trên lưới 2-D <code>(tp, cp)</code> (xem <a href="#fitting-large-models-with-tensor-parallelism">Chạy mô hình lớn với tensor parallelism</a>).</p> <p>Hai yêu cầu sau đây là đặc thù của Cosmos 3:</p> <ul><li>Sử dụng backend attention <code>native</code>. Cosmos 3 sử dụng grouped-query attention (GQA), và backend SDPA native là backend duy nhất chấp nhận <code>enable_gqa</code> (cuDNN và flash từ chối tham số này). Các hàm trợ giúp sẽ mở rộng các đầu KV lên số lượng đầu query và gọi SDPA với <code>enable_gqa=False</code> để nó vẫn được chuyển đến kernel flash (phương án dự phòng về mặt toán học sẽ tạo ra toàn bộ điểm số <code>[S, S]</code> và gây tràn bộ nhớ trên các chuỗi dài).</li> <li>Bậc CP (Ulysses) phải chia hết cho số lượng đầu query (32 với <code>Nano</code>, 64 với <code>Super</code>); với TP, bậc phải chia hết cho số đầu KV (8). Các luồng hiểu (văn bản) và sinh (video/âm thanh) được chia độc lập dọc theo chuỗi, và các độ dài không đều được đệm bằng số 0 bên trong đến một bội số của kích thước world.</li></ul> <!--[2--><h3 class="relative group"><a id="run-it" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#run-it"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Chạy<!----></span></h3><!--]--><!----> <p>CLI đầy đủ <a href="https://github.com/huggingface/diffusers/blob/main/examples/cosmos3/inference_cosmos3.py" rel="nofollow"><code>examples/cosmos3/inference_cosmos3.py</code></a> sử dụng <a href="/docs/diffusers/main/vi/api/pipelines/cosmos3#diffusers.Cosmos3OmniModularPipeline">Cosmos3OmniModularPipeline</a> và tái sử dụng các hàm trợ giúp này, do đó <strong>mọi chế độ</strong> (text-to-image/video, image-to-video, âm thanh, chế độ hành động) đều chạy đa GPU thông qua <code>--tp-degree</code> / <code>--cp-degree</code>. Khởi chạy bằng <a href="https://docs.pytorch.org/docs/stable/elastic/run.html" rel="nofollow">torchrun</a>; <code>--tp-degree * --cp-degree</code> phải bằng <code>--nproc_per_node</code>. Mọi rank đều tạo ra cùng một kết quả; rank 0 ghi kết quả đó.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Đã sao chép</div><!----></button><!----></div> <pre class="language-bash "><!----><span class="hljs-comment"># CP only — Nano (fits one GPU); CP degree must divide 32 query heads.</span> | |
| torchrun --nproc_per_node=4 examples/cosmos3/inference_cosmos3.py --model nano --cp-degree 4 --prompt <span class="hljs-string">"..."</span> | |
| <span class="hljs-comment"># TP only — Super; TP degree must divide 64 query heads and 8 KV heads.</span> | |
| torchrun --nproc_per_node=4 examples/cosmos3/inference_cosmos3.py --model super --tp-degree 4 --prompt <span class="hljs-string">"..."</span> | |
| <span class="hljs-comment"># TP + CP — Super, with sound (TP=2 x CP=2 across 4 GPUs).</span> | |
| torchrun --nproc_per_node=4 examples/cosmos3/inference_cosmos3.py \ | |
| --model super --tp-degree 2 --cp-degree 2 --enable-sound --prompt <span class="hljs-string">"..."</span><!----></pre></div><!----> <p><code>Super</code> có ~120 GB trọng số, không vừa trên một GPU 96 GB, nên cần TP; <code>Nano</code> vừa trên một GPU duy nhất, nên CP cho nó là tối ưu hóa độ trễ thuần túy. (Bỏ qua cả hai cờ để chạy trên một GPU.)</p> <!--[2--><h3 class="relative group"><a id="fitting-large-models-with-tensor-parallelism" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#fitting-large-models-with-tensor-parallelism"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Đào tạo các mô hình lớn với song song tensor<!----></span></h3><!--]--><!----> <p>CP chia nhỏ <em>activations</em> nhưng nhân bản toàn bộ trọng số trên mỗi rank, do đó không làm giảm dung lượng trọng số của mô hình — một mô hình không vừa trên một GPU vẫn sẽ không vừa chỉ với CP. Để chia nhỏ <strong>trọng số</strong>, <code>enable_cosmos3_tensor_parallel(transformer, tp_mesh)</code> áp dụng tensor parallelism kiểu Megatron trên một trục mesh thứ hai, trực giao:</p> <ul><li>Các phép chiếu attention và MLP được chia cột/hàng trên nhóm TP (<code>to_q/to_k/to_v</code> + <code>add_q/k/v</code> và <code>gate/up</code> của các MLP là song song theo cột; <code>to_out/to_add_out</code> và <code>down</code> của các MLP là song song theo hàng kèm all-reduce). Mỗi rank cuối cùng sở hữu <code>query_heads / tp</code> đầu query và <code>kv_heads / tp</code> đầu KV.</li> <li>TP kết hợp với CP trên lưới thiết bị 2 chiều <code>(tp, cp)</code>: TP tách đầu/cân nặng một cách bền vững, còn CP chia nhỏ chuỗi phía trên. Các ràng buộc là <code>tp</code> phải chia hết cho số đầu KV (8), và <code>tp * cp</code> phải chia hết cho số đầu truy vấn (32 với <code>Nano</code>, 64 với <code>Super</code>).</li> <li>Trọng số được nạp vào CPU và chia nhỏ lên các GPU theo từng lớp, do đó mô hình đầy đủ không bao giờ được tạo ra trên một thiết bị duy nhất.</li></ul> <blockquote class="tip"><p>TP thực hiện all-reduce trên mỗi khối attention và MLP, do đó tiêu tốn nhiều băng thông. Trên các máy chủ không có NVLink, đây là chi phí chiếm ưu thế; hãy ưu tiên mức độ TP nhỏ nhất sao cho trọng số vừa đủ và đưa các GPU còn lại vào CP.</p></blockquote> <!--[2--><h3 class="relative group"><a id="use-it-in-your-own-modular-pipeline" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#use-it-in-your-own-modular-pipeline"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Sử dụng trong pipeline mô-đun của riêng bạn<!----></span></h3><!--]--><!----> <p>Các cờ CLI rất tiện lợi, nhưng bạn có thể gọi trực tiếp các hàm trợ giúp bằng <a href="/docs/diffusers/main/vi/api/pipelines/cosmos3#diffusers.Cosmos3OmniModularPipeline">Cosmos3OmniModularPipeline</a>. Tải cấu hình pipeline và các thành phần lên CPU, áp dụng TP <em>trước khi</em> di chuyển pipeline sang GPU cục bộ của rank, chuyển sang backend <code>native</code>, sau đó bật CP. Không sử dụng <code>device_map</code> cho luồng này:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Đã sao chép</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> os | |
| <span class="hljs-keyword">import</span> sys | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">import</span> torch.distributed <span class="hljs-keyword">as</span> dist | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniModularPipeline | |
| <span class="hljs-keyword">from</span> diffusers.schedulers.scheduling_unipc_multistep <span class="hljs-keyword">import</span> UniPCMultistepScheduler | |
| <span class="hljs-keyword">from</span> torch.distributed.device_mesh <span class="hljs-keyword">import</span> init_device_mesh | |
| <span class="hljs-comment"># Make the helper module importable.</span> | |
| sys.path.insert(<span class="hljs-number">0</span>, <span class="hljs-string">"examples/cosmos3"</span>) | |
| <span class="hljs-keyword">from</span> cosmos_parallel <span class="hljs-keyword">import</span> ( | |
| enable_cosmos3_context_parallel, | |
| enable_cosmos3_flash_attention, | |
| enable_cosmos3_tensor_parallel, | |
| ) | |
| <span class="hljs-comment"># torchrun sets RANK / WORLD_SIZE / LOCAL_RANK. Pick tp_degree * cp_degree == world size.</span> | |
| local_rank = <span class="hljs-built_in">int</span>(os.environ[<span class="hljs-string">"LOCAL_RANK"</span>]) | |
| torch.cuda.set_device(local_rank) | |
| dist.init_process_group(<span class="hljs-string">"nccl"</span>) | |
| mesh = init_device_mesh(<span class="hljs-string">"cuda"</span>, (tp_degree, cp_degree), mesh_dim_names=(<span class="hljs-string">"tp"</span>, <span class="hljs-string">"cp"</span>)) | |
| <span class="hljs-comment"># Load components on CPU first; a TP-sharded model may not fit one GPU.</span> | |
| pipe = Cosmos3OmniModularPipeline.from_pretrained(model_id) | |
| pipe.load_components(dtype=torch.bfloat16) | |
| pipe.enable_safety_checker() | |
| <span class="hljs-keyword">if</span> tp_degree > <span class="hljs-number">1</span>: | |
| enable_cosmos3_tensor_parallel(pipe.transformer, mesh[<span class="hljs-string">"tp"</span>]) <span class="hljs-comment"># shard weights -> GPUs</span> | |
| pipe.to(<span class="hljs-string">f"cuda:<span class="hljs-subst">{local_rank}</span>"</span>) <span class="hljs-comment"># move the replicated remainder</span> | |
| pipe.transformer.set_attention_backend(<span class="hljs-string">"native"</span>) | |
| <span class="hljs-keyword">if</span> cp_degree > <span class="hljs-number">1</span>: | |
| enable_cosmos3_context_parallel(pipe.transformer, mesh[<span class="hljs-string">"cp"</span>]) <span class="hljs-comment"># shard the sequence</span> | |
| <span class="hljs-keyword">elif</span> tp_degree > <span class="hljs-number">1</span>: | |
| enable_cosmos3_flash_attention(pipe.transformer) <span class="hljs-comment"># GQA-safe dense attention</span> | |
| <span class="hljs-comment"># Modular pipelines replace components through update_components().</span> | |
| scheduler = UniPCMultistepScheduler.from_config( | |
| pipe.scheduler.config, flow_shift=<span class="hljs-number">10.0</span>, use_karras_sigmas=<span class="hljs-literal">False</span> | |
| ) | |
| pipe.update_components(scheduler=scheduler) | |
| <span class="hljs-comment"># A single output name returns that value; a list returns a dictionary.</span> | |
| outputs = pipe( | |
| prompt=<span class="hljs-string">'{"scene":"A robot arm in a kitchen"}'</span>, | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| output=[<span class="hljs-string">"videos"</span>, <span class="hljs-string">"sound"</span>, <span class="hljs-string">"sampling_rate"</span>, <span class="hljs-string">"action"</span>], | |
| ) | |
| videos = outputs[<span class="hljs-string">"videos"</span>] | |
| sound = outputs[<span class="hljs-string">"sound"</span>] <span class="hljs-comment"># None unless sound generation was requested.</span> | |
| action = outputs[<span class="hljs-string">"action"</span>] <span class="hljs-comment"># None unless an action workflow produced actions.</span><!----></pre></div><!----> <p>Chỉ dành cho CP (không chia shard trọng số), hãy sử dụng lưới 1-D: <code>init_device_mesh("cuda", (world_size,), mesh_dim_names=("cp",))</code> và chỉ cần <code>enable_cosmos3_context_parallel</code>.</p> <p><code>enable_safety_checker()</code> tải và bật trình kiểm tra mặc định; <code>disable_safety_checker()</code> tắt nó một cách tường minh. Hãy sử dụng các phương thức pipeline đó thay vì tham số khởi tạo <code>enable_safety_checker=</code> của pipeline tác vụ hoặc tham số gọi <code>enable_safety_check=</code>. Các pipeline dạng module cũng không trả về <code>Cosmos3OmniPipelineOutput</code>: hãy dùng <code>output="videos"</code> cho riêng các khung hình, hoặc dùng danh sách output và từ điển được trả về như minh họa ở trên thay vì <code>result.video</code>, <code>result.sound</code> hoặc <code>result.action</code>.</p> <blockquote class="tip"><p>Trên một số cấu hình đa GPU, lệnh all-to-all NCCL đầu tiên có thể bị treo. Nếu một lần chạy CP bị dừng ở đầu bước khử nhiễu đầu tiên, hãy đặt <code>NCCL_P2P_DISABLE=1</code> trong môi trường trước khi khởi chạy <code>torchrun</code>.</p></blockquote> <p>CP và TP kết hợp với tất cả các quy trình làm việc ở trên (text-to-video, image-to-video, text-to-video có âm thanh, và sinh có điều kiện hành động) cũng như với cả checkpoint <code>Nano</code> và <code>Super</code> — chỉ có các dòng xây dựng pipeline và thiết lập song song là thay đổi.</p> <!--[1--><h2 class="relative group"><a id="metadata-templates" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#metadata-templates"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Các mẫu metadata<!----></span></h2><!--]--><!----> <p><code>tokenize_prompt</code> thêm các câu metadata ngắn vào bên trong thông điệp của người dùng để LLM nhìn thấy các điều kiện mà mô hình đã được huấn luyện. Prompt tích cực nhận các câu như <em>“The video is 7.9 seconds long and is of 24 FPS.”</em> và <em>“This video is of 720x1280 resolution.”</em>; prompt tiêu cực nhận phần ngược lại (<em>”… is not …”</em>).</p> <p>Cả hai đều được bật theo mặc định. Tắt từng cặp thông qua <code>__call__</code>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Đã sao chép</div><!----></button><!----></div> <pre class="language-python "><!---->result = pipe( | |
| prompt=prompt, | |
| negative_prompt=negative_prompt, | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| fps=<span class="hljs-number">24.0</span>, | |
| add_duration_template=<span class="hljs-literal">False</span>, <span class="hljs-comment"># skip the duration sentence on both prompts</span> | |
| add_resolution_template=<span class="hljs-literal">False</span>, <span class="hljs-comment"># skip the resolution sentence on both prompts</span> | |
| )<!----></pre></div><!----> <p><code>add_duration_template</code> không có tác dụng khi <code>num_frames == 1</code> (chế độ ảnh); trong trường hợp đó, chỉ có câu về độ phân giải được thêm vào.</p> <!--[1--><h2 class="relative group"><a id="safety-checker" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#safety-checker"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Trình kiểm tra an toàn<!----></span></h2><!--]--><!----> <p>Cosmos3 tích hợp <a href="https://pypi.org/project/cosmos-guardrail/" rel="nofollow"><code>cosmos_guardrail</code></a> <code>CosmosSafetyChecker</code> và chạy nó <strong>mặc định</strong>. Guardrail văn bản từ chối các prompt không an toàn trước khi sinh nội dung (<code>ValueError</code>); guardrail video chạy trên các khung hình đã giải mã và hoặc làm mờ các khuôn mặt được phát hiện hoặc từ chối đầu ra. Đầu ra âm thanh không được bảo vệ bởi guardrail.</p> <p>Cài đặt phụ thuộc tùy chọn để bật trình kiểm tra mặc định:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Đã sao chép</div><!----></button><!----></div> <pre class=" "><!---->pip <span class="hljs-keyword">install</span> cosmos_guardrail<!----></pre></div><!----> <p>Trình kiểm tra là bắt buộc theo Thỏa thuận Giấy phép Mô hình Mở của NVIDIA. Hai cờ bên dưới tồn tại cho các quy trình kiểm thử và phát triển, nơi rào chắn sẽ là thừa (ví dụ: đầu vào đã được làm sạch, hoặc bạn đang cố ý chạy thử pipeline trên các đầu vào biên).</p> <p><strong>Tắt khi khởi tạo</strong> (không có trình kiểm tra nào được khởi tạo, do đó không có mô hình guardrail nào được tải xuống hoặc nạp vào bộ nhớ):</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Đã sao chép</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniPipeline | |
| pipe = Cosmos3OmniPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Nano"</span>, | |
| dtype=torch.bfloat16, | |
| device_map=<span class="hljs-string">"cuda"</span>, <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| enable_safety_checker=<span class="hljs-literal">False</span>, | |
| )<!----></pre></div><!----> <p><strong>Tắt cho một lần gọi</strong> (trình kiểm tra vẫn được nạp — hữu ích cho việc bỏ qua một lần duy nhất trong khi giữ mặc định bật cho các lần gọi sau):</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Đã sao chép</div><!----></button><!----></div> <pre class="language-python "><!---->result = pipe( | |
| prompt=prompt, | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| fps=<span class="hljs-number">24.0</span>, | |
| enable_safety_check=<span class="hljs-literal">False</span>, | |
| )<!----></pre></div><!----> <p>Để cung cấp một trình kiểm tra tùy chỉnh (ví dụ: một lớp con không thực hiện gì cho các bài kiểm tra nhanh), hãy truyền nó dưới dạng <code>safety_checker=</code>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Đã sao chép</div><!----></button><!----></div> <pre class="language-python "><!---->pipe = Cosmos3OmniPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Nano"</span>, | |
| dtype=torch.bfloat16, | |
| device_map=<span class="hljs-string">"cuda"</span>, <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| safety_checker=MyCustomSafetyChecker(), | |
| )<!----></pre></div><!----> <!--[1--><h2 class="relative group"><a id="diffusers.Cosmos3OmniPipeline" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#diffusers.Cosmos3OmniPipeline"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Cosmos3OmniPipeline<!----></span></h2><!--]--><!----> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><div><span class="group flex space-x-1.5 items-center text-gray-800 bg-gradient-to-r rounded-tr-lg -mt-4 -ml-4 pt-3 px-2.5" id="diffusers.Cosmos3OmniPipeline"><!----><h3 class="!m-0"><span class="flex-1 break-all md:text-lg bg-gradient-to-r px-2.5 py-1.5 rounded-xl from-indigo-50/70 to-white dark:from-gray-900 dark:to-gray-950 dark:text-indigo-300 text-indigo-700"><svg class="mr-1.5 text-indigo-500 inline-block -mt-0.5" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width=".8em" height=".8em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 24 24"><path class="uim-quaternary" d="M20.23 7.24L12 12L3.77 7.24a1.98 1.98 0 0 1 .7-.71L11 2.76c.62-.35 1.38-.35 2 0l6.53 3.77c.29.173.531.418.7.71z" opacity=".25" fill="currentColor"></path><path class="uim-tertiary" d="M12 12v9.5a2.09 2.09 0 0 1-.91-.21L4.5 17.48a2.003 2.003 0 0 1-1-1.73v-7.5a2.06 2.06 0 0 1 .27-1.01L12 12z" opacity=".5" fill="currentColor"></path><path class="uim-primary" d="M20.5 8.25v7.5a2.003 2.003 0 0 1-1 1.73l-6.62 3.82c-.275.13-.576.198-.88.2V12l8.23-4.76c.175.308.268.656.27 1.01z" fill="currentColor"></path></svg><span class="font-light">lớp</span> <span class="font-medium">diffusers.</span><span class="font-semibold">Cosmos3OmniPipeline</span></span></h3><!----> <a id="diffusers.Cosmos3OmniPipeline" class="header-link invisible with-hover:group-hover:visible pr-2" href="#diffusers.Cosmos3OmniPipeline"><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></a> <!--[0--><a class="!ml-auto !text-gray-400 !no-underline text-sm flex items-center" href="https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/cosmos/pipeline_cosmos3_omni.py#L413" target="_blank"><span><</span> <span class="hidden md:block mx-0.5 hover:!underline">source</span> <span>></span></a><!--]--></span> <!--[0--><p class="font-mono text-xs md:text-sm !leading-relaxed !my-6"><span>(</span> <!--[--><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">transformer<span class="opacity-60">: Cosmos3OmniTransformer</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">text_tokenizer<span class="opacity-60">: AutoTokenizer</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">vae<span class="opacity-60">: AutoencoderKLWan</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">scheduler<span class="opacity-60">: UniPCMultistepScheduler</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">sound_tokenizer<span class="opacity-60">: diffusers.models.autoencoders.autoencoder_cosmos3_audio.Cosmos3AVAEAudioTokenizer | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">safety_checker<span class="opacity-60">: diffusers.pipelines.cosmos.pipeline_cosmos3_omni.CosmosSafetyChecker | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">enable_safety_checker<span class="opacity-60">: bool = True</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">default_use_system_prompt<span class="opacity-60">: bool = True</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">use_native_flow_schedule<span class="opacity-60">: bool = False</span></span></span><!--]--> <span>)</span> <!--[-1--><!--]--></p><!--]--> <div class="!mb-10 relative docstring-details "><!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--></div></div><!----> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><div><span class="group flex space-x-1.5 items-center text-gray-800 bg-gradient-to-r rounded-tr-lg -mt-4 -ml-4 pt-3 px-2.5" id="diffusers.Cosmos3OmniPipeline.decode_sound"><!----><h4 class="!m-0"><span class="flex-1 rounded-xl py-0.5 break-all bg-gradient-to-r from-blue-50/60 to-white dark:from-gray-900 dark:to-gray-950 text-blue-700 dark:text-blue-300 font-medium px-2"><svg width="1em" height="1em" viewBox="0 0 32 33" class="mr-1 inline-block -mt-0.5" xmlns="http://www.w3.org/2000/svg"><path d="M5.80566 18.3545C4.90766 17.4565 4.90766 16.0005 5.80566 15.1025L14.3768 6.53142C15.2748 5.63342 16.7307 5.63342 17.6287 6.53142L26.1999 15.1025C27.0979 16.0005 27.0979 17.4565 26.1999 18.3545L17.6287 26.9256C16.7307 27.8236 15.2748 27.8236 14.3768 26.9256L5.80566 18.3545Z" fill="currentColor" fill-opacity="0.25"/><path fill-rule="evenodd" clip-rule="evenodd" d="M16.4801 13.9619C16.4801 12.9761 16.7467 12.5436 16.9443 12.3296C17.1764 12.078 17.5731 11.8517 18.2275 11.707C18.8821 11.5623 19.638 11.5342 20.4038 11.5582C20.7804 11.57 21.1341 11.5932 21.4719 11.6156L21.5263 11.6193C21.8195 11.6389 22.1626 11.6618 22.4429 11.6618V7.40825C22.3209 7.40825 22.1219 7.39596 21.7544 7.37149C21.4202 7.34925 20.9976 7.32115 20.5371 7.30672C19.6286 7.27824 18.4672 7.29779 17.3093 7.55377C16.1512 7.8098 14.8404 8.33724 13.8181 9.4452C12.7612 10.5907 12.2266 12.1236 12.2266 13.9619V15.0127H10.6836V19.2662H12.2266V26.6332H16.4801V19.2662H20.3394V15.0127H16.4801V13.9619Z" fill="currentColor"/></svg>decode_sound</span></h4><!----> <a id="diffusers.Cosmos3OmniPipeline.decode_sound" class="header-link invisible with-hover:group-hover:visible pr-2" href="#diffusers.Cosmos3OmniPipeline.decode_sound"><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></a> <!--[0--><a class="!ml-auto !text-gray-400 !no-underline text-sm flex items-center" href="https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/cosmos/pipeline_cosmos3_omni.py#L517" target="_blank"><span><</span> <span class="hidden md:block mx-0.5 hover:!underline">source</span> <span>></span></a><!--]--></span> <!--[0--><p class="font-mono text-xs md:text-sm !leading-relaxed !my-6"><span>(</span> <!--[--><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">latent<span class="opacity-60">: Tensor</span></span></span><!--]--> <span>)</span> <!--[-1--><!--]--></p><!--]--> <div class="!mb-10 relative docstring-details "><!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--></div></div><!----> <p>Giải mã âm thanh tiềm ẩn <code>[C, T]</code> thành dạng sóng <code>[audio_ch, N]</code>.</p> <p>Thêm/xóa chiều batch mà bộ giải mã của bộ phân từ âm thanh kỳ vọng.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><div><span class="group flex space-x-1.5 items-center text-gray-800 bg-gradient-to-r rounded-tr-lg -mt-4 -ml-4 pt-3 px-2.5" id="diffusers.Cosmos3OmniPipeline.prepare_latents"><!----><h4 class="!m-0"><span class="flex-1 rounded-xl py-0.5 break-all bg-gradient-to-r from-blue-50/60 to-white dark:from-gray-900 dark:to-gray-950 text-blue-700 dark:text-blue-300 font-medium px-2"><svg width="1em" height="1em" viewBox="0 0 32 33" class="mr-1 inline-block -mt-0.5" xmlns="http://www.w3.org/2000/svg"><path d="M5.80566 18.3545C4.90766 17.4565 4.90766 16.0005 5.80566 15.1025L14.3768 6.53142C15.2748 5.63342 16.7307 5.63342 17.6287 6.53142L26.1999 15.1025C27.0979 16.0005 27.0979 17.4565 26.1999 18.3545L17.6287 26.9256C16.7307 27.8236 15.2748 27.8236 14.3768 26.9256L5.80566 18.3545Z" fill="currentColor" fill-opacity="0.25"/><path fill-rule="evenodd" clip-rule="evenodd" d="M16.4801 13.9619C16.4801 12.9761 16.7467 12.5436 16.9443 12.3296C17.1764 12.078 17.5731 11.8517 18.2275 11.707C18.8821 11.5623 19.638 11.5342 20.4038 11.5582C20.7804 11.57 21.1341 11.5932 21.4719 11.6156L21.5263 11.6193C21.8195 11.6389 22.1626 11.6618 22.4429 11.6618V7.40825C22.3209 7.40825 22.1219 7.39596 21.7544 7.37149C21.4202 7.34925 20.9976 7.32115 20.5371 7.30672C19.6286 7.27824 18.4672 7.29779 17.3093 7.55377C16.1512 7.8098 14.8404 8.33724 13.8181 9.4452C12.7612 10.5907 12.2266 12.1236 12.2266 13.9619V15.0127H10.6836V19.2662H12.2266V26.6332H16.4801V19.2662H20.3394V15.0127H16.4801V13.9619Z" fill="currentColor"/></svg>prepare_latents</span></h4><!----> <a id="diffusers.Cosmos3OmniPipeline.prepare_latents" class="header-link invisible with-hover:group-hover:visible pr-2" href="#diffusers.Cosmos3OmniPipeline.prepare_latents"><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></a> <!--[0--><a class="!ml-auto !text-gray-400 !no-underline text-sm flex items-center" href="https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/cosmos/pipeline_cosmos3_omni.py#L763" target="_blank"><span><</span> <span class="hidden md:block mx-0.5 hover:!underline">source</span> <span>></span></a><!--]--></span> <!--[0--><p class="font-mono text-xs md:text-sm !leading-relaxed !my-6"><span>(</span> <!--[--><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">image<span class="opacity-60">: typing.Union[PIL.Image.Image, numpy.ndarray, torch.Tensor, NoneType] = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">video<span class="opacity-60">: typing.Union[list[PIL.Image.Image], torch.Tensor, numpy.ndarray, NoneType] = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">condition_frame_indexes_vision<span class="opacity-60">: collections.abc.Iterable[int] = (0, 1)</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">condition_video_keep<span class="opacity-60">: typing.Literal['first', 'last'] = 'first'</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">num_frames<span class="opacity-60">: int | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">height<span class="opacity-60">: int | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">width<span class="opacity-60">: int | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">fps<span class="opacity-60">: float = 24.0</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">latents<span class="opacity-60">: typing.Optional[torch.Tensor] = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">sound_latents<span class="opacity-60">: typing.Optional[torch.Tensor] = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">action_latents<span class="opacity-60">: typing.Optional[torch.Tensor] = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">generator<span class="opacity-60">: typing.Optional[torch.Generator] = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">device<span class="opacity-60">: str = 'cuda'</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">dtype<span class="opacity-60">: dtype = torch.bfloat16</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">enable_sound<span class="opacity-60">: bool = False</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">action<span class="opacity-60">: CosmosActionCondition | None = None</span></span></span><!--]--> <span>)</span> <!--[-1--><!--]--></p><!--]--> <div class="!mb-10 relative docstring-details "><!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--></div></div><!----> <p>Xây dựng điều kiện + nhiễu ban đầu cho một mẫu duy nhất.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><div><span class="group flex space-x-1.5 items-center text-gray-800 bg-gradient-to-r rounded-tr-lg -mt-4 -ml-4 pt-3 px-2.5" id="diffusers.Cosmos3OmniPipeline.tokenize_prompt"><!----><h4 class="!m-0"><span class="flex-1 rounded-xl py-0.5 break-all bg-gradient-to-r from-blue-50/60 to-white dark:from-gray-900 dark:to-gray-950 text-blue-700 dark:text-blue-300 font-medium px-2"><svg width="1em" height="1em" viewBox="0 0 32 33" class="mr-1 inline-block -mt-0.5" xmlns="http://www.w3.org/2000/svg"><path d="M5.80566 18.3545C4.90766 17.4565 4.90766 16.0005 5.80566 15.1025L14.3768 6.53142C15.2748 5.63342 16.7307 5.63342 17.6287 6.53142L26.1999 15.1025C27.0979 16.0005 27.0979 17.4565 26.1999 18.3545L17.6287 26.9256C16.7307 27.8236 15.2748 27.8236 14.3768 26.9256L5.80566 18.3545Z" fill="currentColor" fill-opacity="0.25"/><path fill-rule="evenodd" clip-rule="evenodd" d="M16.4801 13.9619C16.4801 12.9761 16.7467 12.5436 16.9443 12.3296C17.1764 12.078 17.5731 11.8517 18.2275 11.707C18.8821 11.5623 19.638 11.5342 20.4038 11.5582C20.7804 11.57 21.1341 11.5932 21.4719 11.6156L21.5263 11.6193C21.8195 11.6389 22.1626 11.6618 22.4429 11.6618V7.40825C22.3209 7.40825 22.1219 7.39596 21.7544 7.37149C21.4202 7.34925 20.9976 7.32115 20.5371 7.30672C19.6286 7.27824 18.4672 7.29779 17.3093 7.55377C16.1512 7.8098 14.8404 8.33724 13.8181 9.4452C12.7612 10.5907 12.2266 12.1236 12.2266 13.9619V15.0127H10.6836V19.2662H12.2266V26.6332H16.4801V19.2662H20.3394V15.0127H16.4801V13.9619Z" fill="currentColor"/></svg>tokenize_prompt</span></h4><!----> <a id="diffusers.Cosmos3OmniPipeline.tokenize_prompt" class="header-link invisible with-hover:group-hover:visible pr-2" href="#diffusers.Cosmos3OmniPipeline.tokenize_prompt"><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></a> <!--[0--><a class="!ml-auto !text-gray-400 !no-underline text-sm flex items-center" href="https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/cosmos/pipeline_cosmos3_omni.py#L1132" target="_blank"><span><</span> <span class="hidden md:block mx-0.5 hover:!underline">source</span> <span>></span></a><!--]--></span> <!--[0--><p class="font-mono text-xs md:text-sm !leading-relaxed !my-6"><span>(</span> <!--[--><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">prompt<span class="opacity-60">: str</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">negative_prompt<span class="opacity-60">: str | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">num_frames<span class="opacity-60">: int = 189</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">height<span class="opacity-60">: int = 720</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">width<span class="opacity-60">: int = 1280</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">fps<span class="opacity-60">: float = 24.0</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">use_system_prompt<span class="opacity-60">: bool | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">add_resolution_template<span class="opacity-60">: bool = True</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">add_duration_template<span class="opacity-60">: bool = True</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">action_mode<span class="opacity-60">: str | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">action_view_point<span class="opacity-60">: str | None = None</span></span></span><!--]--> <span>)</span> <!--[-1--><!--]--></p><!--]--> <div class="!mb-10 relative docstring-details "><!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--></div></div><!----> <p>Áp dụng các mẫu bổ trợ prompt và thực hiện tokenize các prompt cond/uncond thông qua mẫu chat được cấu hình.</p> <p>Pipeline này không chạy một text encoder riêng biệt: transformer Cosmos3 kết hợp nhận các token ID thô cùng với các token thị giác (và tùy chọn là âm thanh).</p> <p>Khi <code>negative_prompt</code> là <code>None</code>, một chuỗi rỗng sẽ được sử dụng; trang tài liệu của Cosmos3 liệt kê các negative prompt kiểm soát chất lượng được khuyến nghị để truyền vào một cách tường minh cho text2video / image2video. Các mẫu thời lượng và độ phân giải sẽ được nối vào prompt, và các mẫu ngược sẽ được nối vào negative prompt, khi được bật.</p> <p>Khi <code>action_mode</code> được đặt, prompt sẽ được chuyển đổi thành caption JSON hành động có cấu trúc mà mô hình đã được huấn luyện (xem <code>_build_action_json_prompt</code>), sử dụng <code>action_view_point</code> cho trường framing; các mẫu metadata phẳng sẽ bị bỏ qua vì JSON đã chứa duration/fps/resolution/aspect_ratio.</p></div></div> <ul><li>tất cả</li> <li><strong>call</strong></li></ul> <!--[1--><h2 class="relative group"><a id="cosmos3omnimodularpipeline" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#cosmos3omnimodularpipeline"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Cosmos3OmniModularPipeline<!----></span></h2><!--]--><!----> <p>Cosmos 3 cũng có sẵn dưới dạng pipeline Modular Diffusers. Pipeline dựa trên tác vụ <a href="/docs/diffusers/main/vi/api/pipelines/cosmos3#diffusers.Cosmos3OmniPipeline">Cosmos3OmniPipeline</a> vẫn khả dụng; pipeline modular tồn tại song song với nó và bao phủ các chế độ giống nhau (<code>text2image</code>, <code>text2video</code>, <code>image2video</code>, <code>video2video</code>, tạo có điều kiện hành động, và <code>transfer</code> (kiểm soát cấu trúc), kèm âm thanh tùy chọn khi checkpoint hỗ trợ).</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Đã sao chép</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniModularPipeline | |
| pipe = Cosmos3OmniModularPipeline.from_pretrained( | |
| <span class="hljs-string">"nvidia/Cosmos3-Nano"</span>, dtype=torch.bfloat16 | |
| ) | |
| pipe.load_components(dtype=torch.bfloat16) | |
| pipe.enable_safety_checker() | |
| videos = pipe( | |
| prompt=<span class="hljs-string">'{"scene":"A robot arm in a kitchen"}'</span>, | |
| num_frames=<span class="hljs-number">1</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| output=<span class="hljs-string">"videos"</span>, | |
| ) | |
| <span class="hljs-comment"># Modular pipelines expose declared outputs directly instead of using the task pipeline's</span> | |
| <span class="hljs-comment"># `return_dict`/`Cosmos3OmniPipelineOutput` API.</span> | |
| image = videos[<span class="hljs-number">0</span>]<!----></pre></div><!----> <p>Bạn cũng có thể tải thông qua <a href="/docs/diffusers/main/vi/api/modular_diffusers/pipeline#diffusers.ModularPipeline">ModularPipeline</a> và để cấu hình kho chọn lớp blocks:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Đã sao chép</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> ModularPipeline | |
| pipe = ModularPipeline.from_pretrained(<span class="hljs-string">"nvidia/Cosmos3-Nano"</span>, dtype=torch.bfloat16) | |
| pipe.load_components(dtype=torch.bfloat16) | |
| pipe.enable_safety_checker() | |
| videos = pipe( | |
| prompt=<span class="hljs-string">'{"scene":"A robot arm in a kitchen"}'</span>, num_frames=<span class="hljs-number">1</span>, height=<span class="hljs-number">720</span>, width=<span class="hljs-number">1280</span>, output=<span class="hljs-string">"videos"</span> | |
| )<!----></pre></div><!----> <p>Để kiểm tra hoặc tùy chỉnh một quy trình làm việc Cosmos mô-đun cụ thể, hãy sử dụng <code>available_workflows</code> + <code>get_workflow()</code>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Đã sao chép</div><!----></button><!----></div> <pre class="language-python "><!---->available = pipe.blocks.available_workflows | |
| image2video_blocks = pipe.blocks.get_workflow(<span class="hljs-string">"image2video"</span>)<!----></pre></div><!----> <!--[2--><h3 class="relative group"><a id="modular-examples-for-all-existing-workflows" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#modular-examples-for-all-existing-workflows"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Các ví dụ mô-đun cho tất cả các quy trình làm việc hiện có<!----></span></h3><!--]--><!----> <p>Pipeline dạng module hỗ trợ các chữ ký gọi giống như pipeline theo tác vụ. Các đoạn mã bên dưới phản ánh mọi ví dụ sinh được hiển thị ở trên (<code>text2video</code>, <code>text2image</code>, <code>image2video</code>, <code>video2video</code>, <code>video2video_sound</code>, <code>text2video_sound</code>, và <code>action_policy</code>). Transfer (kiểm soát cấu trúc) có các đầu vào riêng và được trình bày riêng trong <a href="#modular-transfer-structural-control">Modular transfer</a> bên dưới.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Đã sao chép</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniModularPipeline, CosmosActionCondition | |
| <span class="hljs-keyword">from</span> diffusers.schedulers.scheduling_unipc_multistep <span class="hljs-keyword">import</span> UniPCMultistepScheduler | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> encode_video, export_to_video, load_image, load_video | |
| pipe = Cosmos3OmniModularPipeline.from_pretrained(<span class="hljs-string">"nvidia/Cosmos3-Nano"</span>, dtype=torch.bfloat16) | |
| pipe.load_components(dtype=torch.bfloat16) | |
| pipe.enable_safety_checker() | |
| pipe.to(<span class="hljs-string">"cuda"</span>) <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| pipe.scheduler = UniPCMultistepScheduler.from_config( | |
| pipe.scheduler.config, flow_shift=<span class="hljs-number">10.0</span>, use_karras_sigmas=<span class="hljs-literal">False</span> | |
| ) | |
| <span class="hljs-comment"># text2video</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_t2v_prompt.json"</span>)) | |
| negative_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/negative_prompt.json"</span>)) | |
| videos = pipe( | |
| prompt=json.dumps(json_prompt), | |
| negative_prompt=json.dumps(negative_prompt), | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| num_inference_steps=<span class="hljs-number">35</span>, | |
| guidance_scale=<span class="hljs-number">6.0</span>, | |
| fps=<span class="hljs-number">24.0</span>, | |
| output=<span class="hljs-string">"videos"</span>, | |
| ) | |
| export_to_video(videos, <span class="hljs-string">"cosmos3_modular_t2v.mp4"</span>, fps=<span class="hljs-number">24</span>, macro_block_size=<span class="hljs-number">1</span>) | |
| <span class="hljs-comment"># text2image</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_t2i_prompt.json"</span>)) | |
| videos = pipe(prompt=json.dumps(json_prompt), num_frames=<span class="hljs-number">1</span>, height=<span class="hljs-number">720</span>, width=<span class="hljs-number">1280</span>, output=<span class="hljs-string">"videos"</span>) | |
| videos[<span class="hljs-number">0</span>].save(<span class="hljs-string">"cosmos3_modular_t2i.jpg"</span>, <span class="hljs-built_in">format</span>=<span class="hljs-string">"JPEG"</span>, quality=<span class="hljs-number">85</span>) | |
| <span class="hljs-comment"># image2video</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_i2v_prompt.json"</span>)) | |
| negative_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/negative_prompt_i2v.json"</span>)) | |
| image = load_image(<span class="hljs-string">"https://github.com/nvidia-cosmos/cosmos-dependencies/releases/download/assets/robot_153.jpg"</span>) | |
| videos = pipe( | |
| prompt=json.dumps(json_prompt), | |
| negative_prompt=json.dumps(negative_prompt), | |
| image=image, | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| fps=<span class="hljs-number">24.0</span>, | |
| output=<span class="hljs-string">"videos"</span>, | |
| ) | |
| export_to_video(videos, <span class="hljs-string">"cosmos3_modular_i2v.mp4"</span>, fps=<span class="hljs-number">24</span>, macro_block_size=<span class="hljs-number">1</span>) | |
| <span class="hljs-comment"># video2video</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_v2v_prompt.json"</span>)) | |
| negative_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/negative_prompt_i2v.json"</span>)) | |
| video = load_video( | |
| <span class="hljs-string">"https://github.com/nvidia-cosmos/cosmos-dependencies/raw/refs/heads/assets/cosmos3/inputs/vision/robot_pouring.mp4"</span> | |
| ) | |
| videos = pipe( | |
| prompt=json.dumps(json_prompt), | |
| negative_prompt=json.dumps(negative_prompt), | |
| video=video, | |
| condition_frame_indexes_vision=[<span class="hljs-number">0</span>, <span class="hljs-number">1</span>], | |
| condition_video_keep=<span class="hljs-string">"first"</span>, | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| num_inference_steps=<span class="hljs-number">35</span>, | |
| guidance_scale=<span class="hljs-number">6.0</span>, | |
| fps=<span class="hljs-number">24.0</span>, | |
| output=<span class="hljs-string">"videos"</span>, | |
| ) | |
| export_to_video(videos, <span class="hljs-string">"cosmos3_modular_v2v.mp4"</span>, fps=<span class="hljs-number">24</span>, macro_block_size=<span class="hljs-number">1</span>) | |
| <span class="hljs-comment"># video2video_sound</span> | |
| outputs = pipe( | |
| prompt=json.dumps(json_prompt), | |
| negative_prompt=json.dumps(negative_prompt), | |
| video=video, | |
| condition_frame_indexes_vision=[<span class="hljs-number">0</span>, <span class="hljs-number">1</span>], | |
| condition_video_keep=<span class="hljs-string">"first"</span>, | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| fps=<span class="hljs-number">24.0</span>, | |
| enable_sound=<span class="hljs-literal">True</span>, | |
| output=[<span class="hljs-string">"videos"</span>, <span class="hljs-string">"sound"</span>, <span class="hljs-string">"sampling_rate"</span>], | |
| ) | |
| encode_video( | |
| outputs[<span class="hljs-string">"videos"</span>], | |
| fps=<span class="hljs-number">24</span>, | |
| audio=outputs[<span class="hljs-string">"sound"</span>], | |
| audio_sample_rate=outputs[<span class="hljs-string">"sampling_rate"</span>], | |
| output_path=<span class="hljs-string">"cosmos3_modular_v2v_with_sound.mp4"</span>, | |
| ) | |
| <span class="hljs-comment"># text2video_sound</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_t2v_sound_prompt.json"</span>)) | |
| negative_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/negative_prompt.json"</span>)) | |
| outputs = pipe( | |
| prompt=json.dumps(json_prompt), | |
| negative_prompt=json.dumps(negative_prompt), | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| fps=<span class="hljs-number">24.0</span>, | |
| enable_sound=<span class="hljs-literal">True</span>, | |
| output=[<span class="hljs-string">"videos"</span>, <span class="hljs-string">"sound"</span>, <span class="hljs-string">"sampling_rate"</span>], | |
| ) | |
| encode_video( | |
| outputs[<span class="hljs-string">"videos"</span>], | |
| fps=<span class="hljs-number">24</span>, | |
| audio=outputs[<span class="hljs-string">"sound"</span>], | |
| audio_sample_rate=outputs[<span class="hljs-string">"sampling_rate"</span>], | |
| output_path=<span class="hljs-string">"cosmos3_modular_t2v_with_sound.mp4"</span>, | |
| ) | |
| <span class="hljs-comment"># action_policy</span> | |
| prompt = <span class="hljs-string">"Put the pot to the left of the purple item."</span> | |
| action_video = load_video( | |
| <span class="hljs-string">"https://github.com/nvidia-cosmos/cosmos-dependencies/raw/refs/heads/assets/cosmos3/inputs/action/bridge_20260501_0.mp4"</span> | |
| ) | |
| outputs = pipe( | |
| prompt=prompt, | |
| action=CosmosActionCondition( | |
| mode=<span class="hljs-string">"policy"</span>, | |
| chunk_size=<span class="hljs-number">16</span>, | |
| domain_name=<span class="hljs-string">"bridge_orig_lerobot"</span>, | |
| resolution_tier=<span class="hljs-number">480</span>, | |
| video=action_video, | |
| view_point=<span class="hljs-string">"ego_view"</span>, | |
| ), | |
| fps=<span class="hljs-number">5</span>, | |
| num_inference_steps=<span class="hljs-number">30</span>, | |
| guidance_scale=<span class="hljs-number">1.0</span>, | |
| use_system_prompt=<span class="hljs-literal">False</span>, | |
| output=[<span class="hljs-string">"videos"</span>, <span class="hljs-string">"action"</span>], | |
| ) | |
| export_to_video(outputs[<span class="hljs-string">"videos"</span>], <span class="hljs-string">"cosmos3_modular_action_policy.mp4"</span>, fps=<span class="hljs-number">5</span>, macro_block_size=<span class="hljs-number">1</span>) | |
| <span class="hljs-keyword">if</span> outputs[<span class="hljs-string">"action"</span>] <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> <span class="hljs-literal">None</span>: | |
| <span class="hljs-keyword">with</span> <span class="hljs-built_in">open</span>(<span class="hljs-string">"cosmos3_modular_action_policy.json"</span>, <span class="hljs-string">"w"</span>) <span class="hljs-keyword">as</span> f: | |
| json.dump(outputs[<span class="hljs-string">"action"</span>][<span class="hljs-number">0</span>].tolist(), f)<!----></pre></div><!----> <!--[2--><h3 class="relative group"><a id="modular-transfer-structural-control" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#modular-transfer-structural-control"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Chuyển giao mô-đun (kiểm soát cấu trúc)<!----></span></h3><!--]--><!----> <p>Transfer làm theo một <strong>video điều khiển đã tính trước</strong> (cạnh, làm mờ, độ sâu, phân đoạn hoặc bản đồ kịch bản thế giới) được truyền qua <code>control_videos=</code> dưới dạng ánh xạ <code>{hint: video}</code>. Nó chỉ dành cho video (không có <code>image</code> / <code>video</code> / <code>action</code> / <code>enable_sound</code>), prompt là chú thích JSON đã được upsampling trước (xem <a href="#prompt-upsampling">Prompt upsampling</a>), và các clip dài được tạo tự hồi quy theo từng khối <code>num_video_frames_per_chunk</code> rồi được ghép tự động. <code>guidance_scale</code> là CFG văn bản thông thường; <code>control_guidance</code> (<code>!= 1.0</code>) ngoài ra còn khuếch đại tín hiệu điều khiển. Các giá trị khởi đầu được khuyến nghị cho từng hint:</p> <table><thead><tr><th>Gợi ý</th><th><code>guidance_scale</code></th><th><code>control_guidance</code></th><th><code>flow_shift</code></th><th>Hình học</th></tr></thead><tbody><tr><td>Viền / Làm mờ / Độ sâu</td><td>3.0</td><td>1.5</td><td>10.0</td><td>121 khung hình @ 30 FPS</td></tr><tr><td>Phân đoạn</td><td>3.0</td><td>2.0</td><td>10.0</td><td>121 khung hình @ 30 FPS</td></tr><tr><td>Kịch bản thế giới (WSM)</td><td>1.0</td><td>3.0</td><td>10.0</td><td>101 khung hình @ 10 FPS</td></tr></tbody></table> <p>Diffusers không cung cấp các tài nguyên điều khiển. Các tài nguyên có sẵn (một video điều khiển + <code>prompt.json</code> tương ứng cho mỗi gợi ý, cùng với <code>negative_prompt.json</code> dùng chung) nằm trong <a href="https://github.com/NVIDIA/cosmos/tree/6e2c69afafd6444c1ca14a021aba93fcc6b7996a/cookbooks/cosmos3/generator/transfer/assets" rel="nofollow">Cosmos cookbook</a>. Đối với ví dụ về cạnh bên dưới, hãy tải chúng vào thư mục <code>assets/</code> cục bộ:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Đã sao chép</div><!----></button><!----></div> <pre class="language-bash "><!---->base=https://github.com/NVIDIA/cosmos/raw/6e2c69afafd6444c1ca14a021aba93fcc6b7996a/cookbooks/cosmos3/generator/transfer/assets | |
| <span class="hljs-built_in">mkdir</span> -p assets/edge | |
| curl -sL <span class="hljs-string">"<span class="hljs-variable">$base</span>/edge/control_edge.mp4"</span> -o assets/edge/control_edge.mp4 | |
| curl -sL <span class="hljs-string">"<span class="hljs-variable">$base</span>/edge/prompt.json"</span> -o assets/edge/prompt.json | |
| curl -sL <span class="hljs-string">"<span class="hljs-variable">$base</span>/negative_prompt.json"</span> -o assets/negative_prompt.json<!----></pre></div><!----> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Đã sao chép</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3OmniModularPipeline | |
| <span class="hljs-keyword">from</span> diffusers.schedulers.scheduling_unipc_multistep <span class="hljs-keyword">import</span> UniPCMultistepScheduler | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video, load_video | |
| pipe = Cosmos3OmniModularPipeline.from_pretrained(<span class="hljs-string">"nvidia/Cosmos3-Nano"</span>, dtype=torch.bfloat16) | |
| pipe.load_components(dtype=torch.bfloat16) | |
| pipe.to(<span class="hljs-string">"cuda"</span>) <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| pipe.scheduler = UniPCMultistepScheduler.from_config( | |
| pipe.scheduler.config, flow_shift=<span class="hljs-number">10.0</span>, use_karras_sigmas=<span class="hljs-literal">False</span> | |
| ) | |
| <span class="hljs-comment"># Downloaded into assets/ from the Cosmos cookbook (see the curl snippet above).</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/edge/prompt.json"</span>)) | |
| negative_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/negative_prompt.json"</span>)) | |
| control_edge = load_video(<span class="hljs-string">"assets/edge/control_edge.mp4"</span>) | |
| videos = pipe( | |
| prompt=json.dumps(json_prompt), | |
| negative_prompt=json.dumps(negative_prompt), | |
| control_videos={<span class="hljs-string">"edge"</span>: control_edge}, | |
| num_frames=<span class="hljs-number">121</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| fps=<span class="hljs-number">30.0</span>, | |
| num_inference_steps=<span class="hljs-number">35</span>, | |
| guidance_scale=<span class="hljs-number">3.0</span>, | |
| control_guidance=<span class="hljs-number">1.5</span>, | |
| output=<span class="hljs-string">"videos"</span>, | |
| ) | |
| export_to_video(videos, <span class="hljs-string">"cosmos3_modular_transfer_edge.mp4"</span>, fps=<span class="hljs-number">30</span>, macro_block_size=<span class="hljs-number">1</span>)<!----></pre></div><!----> <!--[2--><h3 class="relative group"><a id="diffusers.Cosmos3OmniModularPipeline" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#diffusers.Cosmos3OmniModularPipeline"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Chuyển văn bản thành ảnh và ảnh thành video đã được chưng cất (ít bước)<!----></span></h3><!--]--><!----> <p>Các checkpoint distilled ít bước được phục vụ bởi <a href="/docs/diffusers/main/vi/api/pipelines/cosmos3#diffusers.Cosmos3DistilledModularPipeline">Cosmos3DistilledModularPipeline</a> (các khối: <code>Cosmos3DistilledBlocks</code>); <a href="/docs/diffusers/main/vi/api/pipelines/cosmos3#diffusers.Cosmos3OmniModularPipeline">Cosmos3OmniModularPipeline</a> và <a href="/docs/diffusers/main/vi/api/pipelines/cosmos3#diffusers.Cosmos3OmniPipeline">Cosmos3OmniPipeline</a> cơ sở không hỗ trợ chúng. <code>num_inference_steps</code> được cố định bằng độ dài của cấu hình pipeline <code>distilled_sigmas</code> (từ <code>modular_model_index.json</code> của checkpoint) và <code>guidance_scale</code> bị ép về 1.0 vì guidance đã được tích hợp vào trọng số — việc truyền bất kỳ giá trị nào khác cho một trong hai tham số này sẽ gây ra lỗi, và <code>negative_prompt</code> sẽ được cảnh báo và bỏ qua.</p> <p>Các checkpoint đã chưng cất FP8 (<code>revision="fp8"</code>) không khai báo chính sách độ chính xác hỗn hợp, do đó mọi bước đều duy trì W8A8 bản địa.</p> <p>Prompts tuân theo cùng cấu trúc JSON mô tả như các model không được chưng cất, do đó văn bản ngắn phải được upsample trước — hãy sử dụng <code>--mode text2image</code> (T2I) hoặc <code>--mode image2video</code> (I2V) như đã mô tả trong <a href="#prompt-upsampling">Prompt upsampling</a>, sau đó truyền JSON qua <code>json.dumps(...)</code>.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Đã sao chép</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> Cosmos3DistilledModularPipeline | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video, load_image | |
| <span class="hljs-comment"># JSON-upsampled prompt (see "Prompt upsampling" above).</span> | |
| json_prompt = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_t2i_prompt.json"</span>)) | |
| repo = <span class="hljs-string">"nvidia/Cosmos3-Super-Text2Image-4Step"</span> | |
| pipe = Cosmos3DistilledModularPipeline.from_pretrained(repo, dtype=torch.bfloat16) | |
| pipe.load_components(dtype=torch.bfloat16) | |
| pipe.to(<span class="hljs-string">"cuda"</span>) <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| <span class="hljs-comment"># text-to-image (distilled)</span> | |
| videos = pipe( | |
| prompt=json.dumps(json_prompt), | |
| num_frames=<span class="hljs-number">1</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| output=<span class="hljs-string">"videos"</span>, | |
| ) | |
| videos[<span class="hljs-number">0</span>].save(<span class="hljs-string">"cosmos3_distilled_t2i.jpg"</span>, <span class="hljs-built_in">format</span>=<span class="hljs-string">"JPEG"</span>, quality=<span class="hljs-number">85</span>) | |
| <span class="hljs-comment"># image-to-video (distilled) — load the I2V repo instead</span> | |
| <span class="hljs-comment"># JSON-upsampled prompt (see "Prompt upsampling" above); upsampled from the source prompt</span> | |
| <span class="hljs-comment"># "The right robotic hand picks up the red sphere on the shelf."</span> | |
| json_prompt_i2v = json.load(<span class="hljs-built_in">open</span>(<span class="hljs-string">"assets/example_i2v_prompt.json"</span>)) | |
| repo_i2v = <span class="hljs-string">"nvidia/Cosmos3-Super-Image2Video-4Step"</span> | |
| pipe = Cosmos3DistilledModularPipeline.from_pretrained(repo_i2v, dtype=torch.bfloat16) | |
| pipe.load_components(dtype=torch.bfloat16) | |
| pipe.to(<span class="hljs-string">"cuda"</span>) | |
| image = load_image( | |
| <span class="hljs-string">"https://github.com/nvidia-cosmos/cosmos-dependencies/raw/refs/heads/assets/cosmos3/inputs/vision/robot_153.jpg"</span> | |
| ) | |
| videos = pipe( | |
| prompt=json.dumps(json_prompt_i2v), | |
| image=image, | |
| num_frames=<span class="hljs-number">189</span>, | |
| height=<span class="hljs-number">720</span>, | |
| width=<span class="hljs-number">1280</span>, | |
| output=<span class="hljs-string">"videos"</span>, | |
| ) | |
| export_to_video(videos, <span class="hljs-string">"cosmos3_distilled_i2v.mp4"</span>, fps=<span class="hljs-number">24</span>, macro_block_size=<span class="hljs-number">1</span>)<!----></pre></div><!----> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><div><span class="group flex space-x-1.5 items-center text-gray-800 bg-gradient-to-r rounded-tr-lg -mt-4 -ml-4 pt-3 px-2.5" id="diffusers.Cosmos3OmniModularPipeline"><!----><h3 class="!m-0"><span class="flex-1 break-all md:text-lg bg-gradient-to-r px-2.5 py-1.5 rounded-xl from-indigo-50/70 to-white dark:from-gray-900 dark:to-gray-950 dark:text-indigo-300 text-indigo-700"><svg class="mr-1.5 text-indigo-500 inline-block -mt-0.5" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width=".8em" height=".8em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 24 24"><path class="uim-quaternary" d="M20.23 7.24L12 12L3.77 7.24a1.98 1.98 0 0 1 .7-.71L11 2.76c.62-.35 1.38-.35 2 0l6.53 3.77c.29.173.531.418.7.71z" opacity=".25" fill="currentColor"></path><path class="uim-tertiary" d="M12 12v9.5a2.09 2.09 0 0 1-.91-.21L4.5 17.48a2.003 2.003 0 0 1-1-1.73v-7.5a2.06 2.06 0 0 1 .27-1.01L12 12z" opacity=".5" fill="currentColor"></path><path class="uim-primary" d="M20.5 8.25v7.5a2.003 2.003 0 0 1-1 1.73l-6.62 3.82c-.275.13-.576.198-.88.2V12l8.23-4.76c.175.308.268.656.27 1.01z" fill="currentColor"></path></svg><span class="font-light">lớp</span> <span class="font-medium">diffusers.</span><span class="font-semibold">Cosmos3OmniModularPipeline</span></span></h3><!----> <a id="diffusers.Cosmos3OmniModularPipeline" class="header-link invisible with-hover:group-hover:visible pr-2" href="#diffusers.Cosmos3OmniModularPipeline"><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></a> <!--[0--><a class="!ml-auto !text-gray-400 !no-underline text-sm flex items-center" href="https://github.com/huggingface/diffusers/blob/main/src/diffusers/modular_pipelines/cosmos/modular_pipeline.py#L7" target="_blank"><span><</span> <span class="hidden md:block mx-0.5 hover:!underline">source</span> <span>></span></a><!--]--></span> <!--[0--><p class="font-mono text-xs md:text-sm !leading-relaxed !my-6"><span>(</span> <!--[--><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">blocks<span class="opacity-60">: diffusers.modular_pipelines.modular_pipeline.ModularPipelineBlocks | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">pretrained_model_name_or_path<span class="opacity-60">: str | os.PathLike | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">components_manager<span class="opacity-60">: diffusers.modular_pipelines.components_manager.ComponentsManager | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">collection<span class="opacity-60">: str | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">workflow<span class="opacity-60">: str | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">modular_config_dict<span class="opacity-60">: dict[str, typing.Any] | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">config_dict<span class="opacity-60">: dict[str, typing.Any] | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">**kwargs<span class="opacity-60"></span></span></span><!--]--> <span>)</span> <!--[-1--><!--]--></p><!--]--> <div class="!mb-10 relative docstring-details "><!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--></div></div><!----> <p>Một ModularPipeline cho việc tạo nội dung omni của Cosmos 3.</p></div> <ul><li>tất cả</li> <li><strong>call</strong></li></ul> <!--[1--><h2 class="relative group"><a id="diffusers.Cosmos3DistilledModularPipeline" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#diffusers.Cosmos3DistilledModularPipeline"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Cosmos3DistilledModularPipeline<!----></span></h2><!--]--><!----> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><div><span class="group flex space-x-1.5 items-center text-gray-800 bg-gradient-to-r rounded-tr-lg -mt-4 -ml-4 pt-3 px-2.5" id="diffusers.Cosmos3DistilledModularPipeline"><!----><h3 class="!m-0"><span class="flex-1 break-all md:text-lg bg-gradient-to-r px-2.5 py-1.5 rounded-xl from-indigo-50/70 to-white dark:from-gray-900 dark:to-gray-950 dark:text-indigo-300 text-indigo-700"><svg class="mr-1.5 text-indigo-500 inline-block -mt-0.5" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width=".8em" height=".8em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 24 24"><path class="uim-quaternary" d="M20.23 7.24L12 12L3.77 7.24a1.98 1.98 0 0 1 .7-.71L11 2.76c.62-.35 1.38-.35 2 0l6.53 3.77c.29.173.531.418.7.71z" opacity=".25" fill="currentColor"></path><path class="uim-tertiary" d="M12 12v9.5a2.09 2.09 0 0 1-.91-.21L4.5 17.48a2.003 2.003 0 0 1-1-1.73v-7.5a2.06 2.06 0 0 1 .27-1.01L12 12z" opacity=".5" fill="currentColor"></path><path class="uim-primary" d="M20.5 8.25v7.5a2.003 2.003 0 0 1-1 1.73l-6.62 3.82c-.275.13-.576.198-.88.2V12l8.23-4.76c.175.308.268.656.27 1.01z" fill="currentColor"></path></svg><span class="font-light">lớp</span> <span class="font-medium">diffusers.</span><span class="font-semibold">Cosmos3DistilledModularPipeline</span></span></h3><!----> <a id="diffusers.Cosmos3DistilledModularPipeline" class="header-link invisible with-hover:group-hover:visible pr-2" href="#diffusers.Cosmos3DistilledModularPipeline"><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></a> <!--[0--><a class="!ml-auto !text-gray-400 !no-underline text-sm flex items-center" href="https://github.com/huggingface/diffusers/blob/main/src/diffusers/modular_pipelines/cosmos/modular_pipeline.py#L121" target="_blank"><span><</span> <span class="hidden md:block mx-0.5 hover:!underline">source</span> <span>></span></a><!--]--></span> <!--[0--><p class="font-mono text-xs md:text-sm !leading-relaxed !my-6"><span>(</span> <!--[--><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">blocks<span class="opacity-60">: diffusers.modular_pipelines.modular_pipeline.ModularPipelineBlocks | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">pretrained_model_name_or_path<span class="opacity-60">: str | os.PathLike | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">components_manager<span class="opacity-60">: diffusers.modular_pipelines.components_manager.ComponentsManager | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">collection<span class="opacity-60">: str | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">workflow<span class="opacity-60">: str | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">modular_config_dict<span class="opacity-60">: dict[str, typing.Any] | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">config_dict<span class="opacity-60">: dict[str, typing.Any] | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">**kwargs<span class="opacity-60"></span></span></span><!--]--> <span>)</span> <!--[-1--><!--]--></p><!--]--> <div class="!mb-10 relative docstring-details "><!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--></div></div><!----> <p>ModularPipeline cho việc tạo nội dung Cosmos 3 omni đã được chưng cất (ít bước).</p> <p>Các checkpoint đã được chưng cất tích hợp classifier-free guidance vào trọng số và lấy mẫu theo lịch cố định được đọc từ cấu hình <code>distilled_sigmas</code> của pipeline (được nạp từ <code>modular_model_index.json</code>), do đó <code>guidance_scale</code> và <code>num_inference_steps</code> là cố định và <code>negative_prompt</code> không được hỗ trợ.</p></div> <ul><li>tất cả</li> <li><strong>call</strong></li></ul> <!--[1--><h2 class="relative group"><a id="diffusers.CosmosActionCondition" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#diffusers.CosmosActionCondition"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->CosmosActionCondition<!----></span></h2><!--]--><!----> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><div><span class="group flex space-x-1.5 items-center text-gray-800 bg-gradient-to-r rounded-tr-lg -mt-4 -ml-4 pt-3 px-2.5" id="diffusers.CosmosActionCondition"><!----><h3 class="!m-0"><span class="flex-1 break-all md:text-lg bg-gradient-to-r px-2.5 py-1.5 rounded-xl from-indigo-50/70 to-white dark:from-gray-900 dark:to-gray-950 dark:text-indigo-300 text-indigo-700"><svg class="mr-1.5 text-indigo-500 inline-block -mt-0.5" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width=".8em" height=".8em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 24 24"><path class="uim-quaternary" d="M20.23 7.24L12 12L3.77 7.24a1.98 1.98 0 0 1 .7-.71L11 2.76c.62-.35 1.38-.35 2 0l6.53 3.77c.29.173.531.418.7.71z" opacity=".25" fill="currentColor"></path><path class="uim-tertiary" d="M12 12v9.5a2.09 2.09 0 0 1-.91-.21L4.5 17.48a2.003 2.003 0 0 1-1-1.73v-7.5a2.06 2.06 0 0 1 .27-1.01L12 12z" opacity=".5" fill="currentColor"></path><path class="uim-primary" d="M20.5 8.25v7.5a2.003 2.003 0 0 1-1 1.73l-6.62 3.82c-.275.13-.576.198-.88.2V12l8.23-4.76c.175.308.268.656.27 1.01z" fill="currentColor"></path></svg><span class="font-light">lớp</span> <span class="font-medium">diffusers.</span><span class="font-semibold">CosmosActionCondition</span></span></h3><!----> <a id="diffusers.CosmosActionCondition" class="header-link invisible with-hover:group-hover:visible pr-2" href="#diffusers.CosmosActionCondition"><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></a> <!--[0--><a class="!ml-auto !text-gray-400 !no-underline text-sm flex items-center" href="https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/cosmos/pipeline_cosmos3_omni.py#L302" target="_blank"><span><</span> <span class="hidden md:block mx-0.5 hover:!underline">source</span> <span>></span></a><!--]--></span> <!--[0--><p class="font-mono text-xs md:text-sm !leading-relaxed !my-6"><span>(</span> <!--[--><span class="comma cursor-pointer"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">mode<span class="opacity-60">: typing.Literal['policy', 'forward_dynamics', 'inverse_dynamics']</span></span></span><span class="comma cursor-pointer"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">chunk_size<span class="opacity-60">: int</span></span></span><span class="comma cursor-pointer"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">domain_name<span class="opacity-60">: str</span></span></span><span class="comma cursor-pointer"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">resolution_tier<span class="opacity-60">: int = 480</span></span></span><span class="comma cursor-pointer"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">raw_actions<span class="opacity-60">: typing.Optional[torch.Tensor] = None</span></span></span><span class="comma cursor-pointer"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">image<span class="opacity-60">: typing.Union[PIL.Image.Image, numpy.ndarray, torch.Tensor, NoneType] = None</span></span></span><span class="comma cursor-pointer"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">video<span class="opacity-60">: typing.Union[list, numpy.ndarray, torch.Tensor, NoneType] = None</span></span></span><span class="comma cursor-pointer"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">view_point<span class="opacity-60">: str = 'ego_view'</span></span></span><!--]--> <span>)</span> <!--[-1--><!--]--></p><!--]--> <div class="!mb-10 relative docstring-details "><!--[-1--><!--]--> <!--[0--><p class="flex items-center font-semibold !mt-2 !mb-2 text-gray-800">Tham số <span class="flex-auto border-t-2 border-gray-100 dark:border-gray-700 ml-3"></span></p> <ul class="px-2"><!--[--><li class="text-base !pl-4 my-3 rounded "><span class="group flex space-x-1.5 items-start"><a id="diffusers.CosmosActionCondition.mode" class="header-link block pr-0.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#diffusers.CosmosActionCondition.mode"><span><svg class="text-smd" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!----><strong>mode</strong> (<code>str</code>) — | |
| Nhiệm vụ hành động. Một trong số <code>"forward_dynamics"</code> (triển khai video tương lai từ khung hình đầu tiên và chuỗi | |
| <code>raw_actions</code> đã cho), <code>"inverse_dynamics"</code> (suy luận các hành động kết nối các khung hình điều kiện), hoặc | |
| <code>"policy"</code> (triển khai đồng thời video tương lai và các hành động từ khung hình đầu tiên).<!----></span></span></li><li class="text-base !pl-4 my-3 rounded "><span class="group flex space-x-1.5 items-start"><a id="diffusers.CosmosActionCondition.chunk_size" class="header-link block pr-0.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#diffusers.CosmosActionCondition.chunk_size"><span><svg class="text-smd" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!----><strong>chunk_size</strong> (<code>int</code>) — | |
| Số bước chuyển tiếp hành động trong chunk. Video điều kiện tương ứng trải dài <code>chunk_size + 1</code> | |
| khung hình.<!----></span></span></li><li class="text-base !pl-4 my-3 rounded "><span class="group flex space-x-1.5 items-start"><a id="diffusers.CosmosActionCondition.domain_name" class="header-link block pr-0.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#diffusers.CosmosActionCondition.domain_name"><span><svg class="text-smd" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!----><strong>domain_name</strong> (<code>str</code>) — | |
| Lĩnh vực hiện thân (embodiment domain) dùng để chọn trọng số chiếu hành động nhận thức theo lĩnh vực. Phải là một trong các lĩnh vực hiện thân Cosmos 3 đã đăng ký. Tham số này cũng xác định chiều hành động không đệm (unpadded action width) được dùng để cắt các hành động dự đoán, được giải quyết nội bộ từ tên này (xem <code>_EMBODIMENT_TO_RAW_ACTION_DIM</code>).<!----></span></span></li><li class="text-base !pl-4 my-3 rounded "><span class="group flex space-x-1.5 items-start"><a id="diffusers.CosmosActionCondition.resolution_tier" class="header-link block pr-0.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#diffusers.CosmosActionCondition.resolution_tier"><span><svg class="text-smd" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!----><strong>resolution_tier</strong> (<code>int</code>, mặc định là <code>480</code>) — | |
| Phân cấp độ phân giải <em>tier</em> cho điều kiện hành động (một trong số <code>256</code>, <code>480</code>, <code>704</code>, <code>720</code>). Phân cấp này chọn một khung hình có sẵn với tỷ lệ khung hình gần nhất với đầu vào; đầu vào sẽ được thu nhỏ (không bao giờ phóng to) và được đệm vào trong khung hình đó để phục vụ cho việc điều kiện. Đây không phải là kích thước khung hình đầu ra, vốn bám theo nội dung đầu vào. Hãy khớp phân cấp với độ phân giải gốc của đầu vào: một phân cấp thấp hơn sẽ làm mất chi tiết, trong khi một phân cấp cao hơn không bổ sung độ phân giải (không phóng to), lãng phí tài nguyên tính toán cho phần đệm, và gây ra sự không khớp giữa huấn luyện và suy luận, có thể làm giảm chất lượng.<!----></span></span></li><li class="text-base !pl-4 my-3 rounded "><span class="group flex space-x-1.5 items-start"><a id="diffusers.CosmosActionCondition.raw_actions" class="header-link block pr-0.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#diffusers.CosmosActionCondition.raw_actions"><span><svg class="text-smd" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!----><strong>raw_actions</strong> (<code>torch.Tensor</code>, <em>tùy chọn</em>) — | |
| Các vector hành động miền thô có hình dạng <code>[T, raw_action_dim]</code> điều khiển <code>"forward_dynamics"</code>. Các chuỗi ngắn hơn | |
| <code>chunk_size</code> sẽ lặp lại hành động cuối cùng; các chuỗi dài hơn sẽ bị cắt bớt. Các kênh vượt quá | |
| <code>action_dim</code> của mô hình sẽ bị từ chối, và các đầu vào hẹp hơn sẽ được đệm bằng số 0 lên đến <code>action_dim</code>.<!----></span></span></li><li class="text-base !pl-4 my-3 rounded "><span class="group flex space-x-1.5 items-start"><a id="diffusers.CosmosActionCondition.image" class="header-link block pr-0.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#diffusers.CosmosActionCondition.image"><span><svg class="text-smd" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!----><strong>image</strong> (<code>PIL.Image.Image</code>, <code>np.ndarray</code> hoặc <code>torch.Tensor</code>, <em>tùy chọn</em>) — | |
| Khung điều kiện cho <code>"policy"</code> / <code>"forward_dynamics"</code>. Loại trừ lẫn nhau với <code>video</code>.<!----></span></span></li><li class="text-base !pl-4 my-3 rounded "><span class="group flex space-x-1.5 items-start"><a id="diffusers.CosmosActionCondition.video" class="header-link block pr-0.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#diffusers.CosmosActionCondition.video"><span><svg class="text-smd" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!----><strong>video</strong> (<code>list</code>, <code>np.ndarray</code>, hoặc <code>torch.Tensor</code>, <em>tùy chọn</em>) — | |
| Video điều kiện, bắt buộc cho <code>"inverse_dynamics"</code>. Với <code>"policy"</code> / <code>"forward_dynamics"</code>, chỉ khung hình đầu tiên được sử dụng. Loại trừ lẫn nhau với <code>image</code>.<!----></span></span></li><li class="text-base !pl-4 my-3 rounded "><span class="group flex space-x-1.5 items-start"><a id="diffusers.CosmosActionCondition.view_point" class="header-link block pr-0.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#diffusers.CosmosActionCondition.view_point"><span><svg class="text-smd" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!----><strong>view_point</strong> (<code>str</code>, mặc định là <code>"ego_view"</code>) — | |
| Nhãn góc nhìn camera được dùng để điền vào trường <code>cinematography.framing</code> của chú thích hành động. Một trong số | |
| <code>"ego_view"</code>, <code>"third_person_view"</code>, <code>"wrist_view"</code> hoặc <code>"concat_view"</code>. Mô hình hành động được huấn luyện trên | |
| các chú thích JSON có cấu trúc chứa câu mô tả góc nhìn này; một nhãn không được công nhận sẽ làm mất trường framing | |
| (kèm theo cảnh báo).<!----></span></span></li><!--]--></ul><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--></div></div><!----> <p>Nhóm tất cả các đầu vào cần thiết cho một tác vụ tạo có điều kiện hành động của Cosmos 3.</p> <p>Truyền điều này vào <code>Cosmos3OmniPipeline.__call__()</code> thông qua đối số <code>action</code> thay vì các đối số cấp cao <code>image</code> / <code>height</code> / <code>width</code>, vốn được dành riêng cho các lần chạy t2v, i2v.</p></div> <!--[1--><h2 class="relative group"><a id="diffusers.pipelines.cosmos.pipeline_cosmos3_omni.Cosmos3OmniPipelineOutput" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#diffusers.pipelines.cosmos.pipeline_cosmos3_omni.Cosmos3OmniPipelineOutput"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Cosmos3OmniPipelineOutput<!----></span></h2><!--]--><!----> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><div><span class="group flex space-x-1.5 items-center text-gray-800 bg-gradient-to-r rounded-tr-lg -mt-4 -ml-4 pt-3 px-2.5" id="diffusers.pipelines.cosmos.pipeline_cosmos3_omni.Cosmos3OmniPipelineOutput"><!----><h3 class="!m-0"><span class="flex-1 break-all md:text-lg bg-gradient-to-r px-2.5 py-1.5 rounded-xl from-indigo-50/70 to-white dark:from-gray-900 dark:to-gray-950 dark:text-indigo-300 text-indigo-700"><svg class="mr-1.5 text-indigo-500 inline-block -mt-0.5" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width=".8em" height=".8em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 24 24"><path class="uim-quaternary" d="M20.23 7.24L12 12L3.77 7.24a1.98 1.98 0 0 1 .7-.71L11 2.76c.62-.35 1.38-.35 2 0l6.53 3.77c.29.173.531.418.7.71z" opacity=".25" fill="currentColor"></path><path class="uim-tertiary" d="M12 12v9.5a2.09 2.09 0 0 1-.91-.21L4.5 17.48a2.003 2.003 0 0 1-1-1.73v-7.5a2.06 2.06 0 0 1 .27-1.01L12 12z" opacity=".5" fill="currentColor"></path><path class="uim-primary" d="M20.5 8.25v7.5a2.003 2.003 0 0 1-1 1.73l-6.62 3.82c-.275.13-.576.198-.88.2V12l8.23-4.76c.175.308.268.656.27 1.01z" fill="currentColor"></path></svg><span class="font-light">lớp</span> <span class="font-medium">diffusers.pipelines.cosmos.pipeline_cosmos3_omni.</span><span class="font-semibold">Cosmos3OmniPipelineOutput</span></span></h3><!----> <a id="diffusers.pipelines.cosmos.pipeline_cosmos3_omni.Cosmos3OmniPipelineOutput" class="header-link invisible with-hover:group-hover:visible pr-2" href="#diffusers.pipelines.cosmos.pipeline_cosmos3_omni.Cosmos3OmniPipelineOutput"><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></a> <!--[0--><a class="!ml-auto !text-gray-400 !no-underline text-sm flex items-center" href="https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/cosmos/pipeline_cosmos3_omni.py#L283" target="_blank"><span><</span> <span class="hidden md:block mx-0.5 hover:!underline">source</span> <span>></span></a><!--]--></span> <!--[0--><p class="font-mono text-xs md:text-sm !leading-relaxed !my-6"><span>(</span> <!--[--><span class="comma cursor-pointer"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">video<span class="opacity-60">: Any</span></span></span><span class="comma cursor-pointer"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">sound<span class="opacity-60">: typing.Optional[torch.Tensor] = None</span></span></span><span class="comma cursor-pointer"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">action<span class="opacity-60">: list[torch.Tensor] | None = None</span></span></span><!--]--> <span>)</span> <!--[-1--><!--]--></p><!--]--> <div class="!mb-10 relative docstring-details "><!--[-1--><!--]--> <!--[0--><p class="flex items-center font-semibold !mt-2 !mb-2 text-gray-800">Tham số <span class="flex-auto border-t-2 border-gray-100 dark:border-gray-700 ml-3"></span></p> <ul class="px-2"><!--[--><li class="text-base !pl-4 my-3 rounded "><span class="group flex space-x-1.5 items-start"><a id="diffusers.pipelines.cosmos.pipeline_cosmos3_omni.Cosmos3OmniPipelineOutput.video" class="header-link block pr-0.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#diffusers.pipelines.cosmos.pipeline_cosmos3_omni.Cosmos3OmniPipelineOutput.video"><span><svg class="text-smd" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!----><strong>video</strong> — Video được tạo ra. Kiểu chính xác phụ thuộc vào <code>output_type</code> | |
| được truyền vào pipeline: một danh sách các khung hình PIL cho <code>"pil"</code> (mặc định), một <code>np.ndarray</code> có hình dạng <code>[T, H, W, C]</code> cho <code>"np"</code>, một <code>torch.Tensor</code> có hình dạng <code>[T, C, H, W]</code> cho <code>"pt"</code>, hoặc một tensor latent thô | |
| khi <code>output_type="latent"</code>.<!----></span></span></li><li class="text-base !pl-4 my-3 rounded "><span class="group flex space-x-1.5 items-start"><a id="diffusers.pipelines.cosmos.pipeline_cosmos3_omni.Cosmos3OmniPipelineOutput.sound" class="header-link block pr-0.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#diffusers.pipelines.cosmos.pipeline_cosmos3_omni.Cosmos3OmniPipelineOutput.sound"><span><svg class="text-smd" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!----><strong>sound</strong> — Dạng sóng âm thanh đã giải mã có hình dạng <code>[C, N]</code>. <code>None</code> khi | |
| <code>enable_sound=False</code>.<!----></span></span></li><li class="text-base !pl-4 my-3 rounded "><span class="group flex space-x-1.5 items-start"><a id="diffusers.pipelines.cosmos.pipeline_cosmos3_omni.Cosmos3OmniPipelineOutput.action" class="header-link block pr-0.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#diffusers.pipelines.cosmos.pipeline_cosmos3_omni.Cosmos3OmniPipelineOutput.action"><span><svg class="text-smd" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!----><strong>action</strong> — Các token hành động dự đoán. <code>None</code> trừ khi một chế độ hành động dự đoán các hành động.<!----></span></span></li><!--]--></ul><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--></div></div><!----> <p>Dataclass đầu ra cho <a href="/docs/diffusers/main/vi/api/pipelines/cosmos3#diffusers.Cosmos3OmniPipeline">Cosmos3OmniPipeline</a>.</p></div> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/diffusers/blob/main/docs/source/en/api/pipelines/cosmos3.md" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg><!----> <span><span class="underline">Update</span> on GitHub</span></a><!----> <p></p><!--]--><!----><!--]--><!--]--><!--]--> <!--[-1--><!--]--><!--]--> | |
| <script> | |
| { | |
| __sveltekit_gxjg5b = { | |
| base: "/docs/diffusers/main/en", | |
| assets: "/docs/diffusers/main/en" | |
| }; | |
| const element = document.currentScript.parentElement; | |
| Promise.all([ | |
| import("/docs/diffusers/main/en/_app/immutable/entry/start.U6TUchwJ.js"), | |
| import("/docs/diffusers/main/en/_app/immutable/entry/app.3qKhPoX-.js") | |
| ]).then(([kit, app]) => { | |
| kit.start(app, element, { | |
| node_ids: [0, 7], | |
| data: [null,null], | |
| form: null, | |
| error: null | |
| }); | |
| }); | |
| } | |
| </script> | |
Xet Storage Details
- Size:
- 237 kB
- Xet hash:
- 1e0eecd13b419a45d119afbd2fe4f6f7768a70b6089ecd55d8e08bd39de48f70
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.