Buckets:

hf-doc-build
/

doc

Files
xet
hf-doc-build/doc / diffusers /main /ru /api /pipelines /kandinsky_s2v.html
mishig
about 4 hours ago
download
raw
52.2 kB
<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;Kandinsky S2V 1.0&quot;,&quot;local&quot;:&quot;kandinsky-s2v-10&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Чекпоинты&quot;,&quot;local&quot;:&quot;checkpoints&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Использование&quot;,&quot;local&quot;:&quot;usage&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Примечания&quot;,&quot;local&quot;:&quot;notes&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;KandinskyS2VModularPipeline&quot;,&quot;local&quot;:&quot;diffusers.KandinskyS2VModularPipeline&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;KandinskyS2VAutoBlocks&quot;,&quot;local&quot;:&quot;diffusers.KandinskyS2VAutoBlocks&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;KandinskyS2VDistilledModularPipeline&quot;,&quot;local&quot;:&quot;diffusers.KandinskyS2VDistilledModularPipeline&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;KandinskyS2VDistilledAutoBlocks&quot;,&quot;local&quot;:&quot;diffusers.KandinskyS2VDistilledAutoBlocks&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}"/>
		<link href="/docs/diffusers/main/en/_app/immutable/entry/start.D3LnKZ4R.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/BF9UNEM9.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/DPnyVmXs.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/entry/app.C0rBoMxR.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/DSGUT_OW.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/C045MWWx.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/DnVIJfC5.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/DsnmJJEf.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/nodes/0.DTZsP_3B.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/IaDvMxLQ.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/nodes/197.CNZ1bmB6.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/bKdwVYpU.js" rel="modulepreload">
		<!--1jqdiis--><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;Kandinsky S2V 1.0&quot;,&quot;local&quot;:&quot;kandinsky-s2v-10&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Checkpoints&quot;,&quot;local&quot;:&quot;checkpoints&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Usage&quot;,&quot;local&quot;:&quot;usage&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Notes&quot;,&quot;local&quot;:&quot;notes&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;KandinskyS2VModularPipeline&quot;,&quot;local&quot;:&quot;diffusers.KandinskyS2VModularPipeline&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;KandinskyS2VAutoBlocks&quot;,&quot;local&quot;:&quot;diffusers.KandinskyS2VAutoBlocks&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;KandinskyS2VDistilledModularPipeline&quot;,&quot;local&quot;:&quot;diffusers.KandinskyS2VDistilledModularPipeline&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;KandinskyS2VDistilledAutoBlocks&quot;,&quot;local&quot;:&quot;diffusers.KandinskyS2VDistilledAutoBlocks&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}"/><!---->
		<link href="/docs/diffusers/main/en/_app/immutable/assets/0.tn0RQdqM.css" rel="modulepreload"> <!--[--><!--[0--><!--[--><!--[0--><!--[--><p></p> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: inline-end; margin-inline-start: 10px; display: inline-flex; position: relative; z-index: 10;"><div dir="ltr" class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 h-7 max-sm:h-7 px-2 max-sm:px-1.5 text-sm font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0 hover:text-gray-800 dark:hover:text-gray-200"><svg class="sm:size-3.5 size-3" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----></span> <span dir="auto">Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-7 max-sm:h-7 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible sm:size-3.5 size-3 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg><!----></button></div> <!--[-1--><!--]--></div><!----> <!--[0--><h1 class="relative group"><a id="kandinsky-s2v-10" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#kandinsky-s2v-10"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Kandinsky S2V 1.0<!----></span></h1><!--]--><!----> <p>Kandinsky S2V 1.0 генерирует видео аватара на основе опорного кадра и аудиодорожки с речью. Персонаж на кадре анимируется в соответствии с речью — анимируется всё тело, а не только голова. Речь выступает в роли условия, а не результата: модель не генерирует звук, а звуковым сопровождением итогового видео является переданная вами дорожка.</p> <p>Обе ветви проходят через <a href="https://huggingface.co/Qwen/Qwen2.5-Omni-7B" rel="nofollow">Qwen2.5-Omni-7B</a>:
его скрытые состояния кодируют промпт, а аудиобашня кодирует речь, которая
поступает в трансформер через адаптеры кросс-внимания, выровненные по кадрам, так
что каждый латентный кадр обращается только к той речи, которую он покрывает. Клип генерируется
кусками по 5 с (125 кадров при 25 fps), каждый из которых несёт память FramePack от
предыдущих, благодаря чему длинные дорожки остаются согласованными на границах кусков.</p> <p>Kandinsky S2V 1.0 интегрирован только как <a href="../../modular_diffusers/overview">модульный конвейер</a>; стандартный класс <code>DiffusionPipeline</code> для него отсутствует.</p> <!--[1--><h2 class="relative group"><a id="checkpoints" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#checkpoints"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Чекпоинты<!----></span></h2><!--]--><!----> <table><thead><tr><th>Чекпоинт</th><th>Transformer</th><th>Блоки</th><th>Планировщик</th><th>Шаги</th><th>Руководство</th></tr></thead><tbody><tr><td><a href="https://huggingface.co/kandinskylab/Kandinsky-S2V-1.0-Lite-Diffusers" rel="nofollow">Lite</a></td><td>2B</td><td><a href="/docs/diffusers/main/ru/api/pipelines/kandinsky_s2v#diffusers.KandinskyS2VAutoBlocks">KandinskyS2VAutoBlocks</a></td><td><a href="/docs/diffusers/main/ru/api/schedulers/flow_match_euler_discrete#diffusers.FlowMatchEulerDiscreteScheduler">FlowMatchEulerDiscreteScheduler</a></td><td>50</td><td>3.0</td></tr><tr><td><a href="https://huggingface.co/kandinskylab/Kandinsky-S2V-1.0-Pro-Diffusers" rel="nofollow">Pro</a></td><td>21B</td><td><a href="/docs/diffusers/main/ru/api/pipelines/kandinsky_s2v#diffusers.KandinskyS2VAutoBlocks">KandinskyS2VAutoBlocks</a></td><td><a href="/docs/diffusers/main/ru/api/schedulers/flow_match_euler_discrete#diffusers.FlowMatchEulerDiscreteScheduler">FlowMatchEulerDiscreteScheduler</a></td><td>50</td><td>3.0</td></tr><tr><td><a href="https://huggingface.co/kandinskylab/Kandinsky-S2V-1.0-Lite-distill-Diffusers" rel="nofollow">Лёгкая дистилляция</a></td><td>2B</td><td><a href="/docs/diffusers/main/ru/api/pipelines/kandinsky_s2v#diffusers.KandinskyS2VDistilledAutoBlocks">KandinskyS2VDistilledAutoBlocks</a></td><td><a href="/docs/diffusers/main/ru/api/schedulers/piflow#diffusers.PiflowScheduler">PiflowScheduler</a></td><td>16</td><td>off</td></tr><tr><td><a href="https://huggingface.co/kandinskylab/Kandinsky-S2V-1.0-Pro-distill-Diffusers" rel="nofollow">Pro distilled</a></td><td>21B</td><td><a href="/docs/diffusers/main/ru/api/pipelines/kandinsky_s2v#diffusers.KandinskyS2VDistilledAutoBlocks">KandinskyS2VDistilledAutoBlocks</a></td><td><a href="/docs/diffusers/main/ru/api/schedulers/piflow#diffusers.PiflowScheduler">PiflowScheduler</a></td><td>16</td><td>off</td></tr></tbody></table> <p>Дистиллированные контрольные точки настраивают <code>PiflowScheduler</code> с параметрами <code>shift=10.0</code> и <code>n_grid=5</code>, а не с их значениями по умолчанию.</p> <p><code>ModularPipeline.from_pretrained</code> выбирает блоки из контрольной точки. Шаги по умолчанию принимают значения, указанные выше. Classifier-free guidance — это компонент <code>guider</code>, а не аргумент вызова: базовые контрольные точки поставляются с его значением 3.0. Дистиллированные контрольные точки не содержат ни guider, ни <code>negative_prompt</code>, поскольку guidance уже является частью student.</p> <!--[1--><h2 class="relative group"><a id="usage" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#usage"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Использование<!----></span></h2><!--]--><!----> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Скопировано</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">import</span> librosa
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> ModularPipeline
<span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> encode_video, load_image

pipe = ModularPipeline.from_pretrained(<span class="hljs-string">&quot;kandinskylab/Kandinsky-S2V-1.0-Lite-distill-Diffusers&quot;</span>)
pipe.load_components(dtype=torch.bfloat16)
pipe.to(<span class="hljs-string">&quot;cuda&quot;</span>)

image = load_image(<span class="hljs-string">&quot;reference.png&quot;</span>)
audio, _ = librosa.load(<span class="hljs-string">&quot;speech.wav&quot;</span>, sr=<span class="hljs-number">16000</span>)  <span class="hljs-comment"># a mono waveform at 16 kHz, for the model</span>
speech, sample_rate = librosa.load(<span class="hljs-string">&quot;speech.wav&quot;</span>, sr=<span class="hljs-literal">None</span>)  <span class="hljs-comment"># the original track, added to the output video</span>

video = pipe(image=image, audio=audio, output=<span class="hljs-string">&quot;videos&quot;</span>)[<span class="hljs-number">0</span>]
encode_video(
    video,
    fps=<span class="hljs-number">25</span>,
    output_path=<span class="hljs-string">&quot;avatar.mp4&quot;</span>,
    audio=torch.from_numpy(speech)[<span class="hljs-literal">None</span>],
    audio_sample_rate=sample_rate,
)<!----></pre></div><!----> <p>Чекпоинты Pro не помещаются на GPU с 80 ГБ памяти вместе с аудио- и текстовым энкодером 7B, если все компоненты находятся на устройстве. Используйте оффлоад через <code>ComponentsManager</code>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Скопировано</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">import</span> librosa
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> ComponentsManager, ModularPipeline
<span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> encode_video, load_image

manager = ComponentsManager()
pipe = ModularPipeline.from_pretrained(<span class="hljs-string">&quot;kandinskylab/Kandinsky-S2V-1.0-Pro-Diffusers&quot;</span>, components_manager=manager)
pipe.load_components(dtype=torch.bfloat16)
manager.enable_auto_cpu_offload(device=<span class="hljs-string">&quot;cuda&quot;</span>, memory_reserve_margin=<span class="hljs-string">&quot;40GiB&quot;</span>)

image = load_image(<span class="hljs-string">&quot;reference.png&quot;</span>)
audio, _ = librosa.load(<span class="hljs-string">&quot;speech.wav&quot;</span>, sr=<span class="hljs-number">16000</span>)
speech, sample_rate = librosa.load(<span class="hljs-string">&quot;speech.wav&quot;</span>, sr=<span class="hljs-literal">None</span>)

video = pipe(image=image, audio=audio, output=<span class="hljs-string">&quot;videos&quot;</span>)[<span class="hljs-number">0</span>]
encode_video(
    video,
    fps=<span class="hljs-number">25</span>,
    output_path=<span class="hljs-string">&quot;avatar.mp4&quot;</span>,
    audio=torch.from_numpy(speech)[<span class="hljs-literal">None</span>],
    audio_sample_rate=sample_rate,
)<!----></pre></div><!----> <p>Margin — это объём памяти, который остаётся свободным сверх весов при перемещении модели на GPU.
Он должен вмещать VAE-декодирование фрагмента, что составляет около 33 ГиБ при разрешении 512x768: при значении по умолчанию 3 ГБ
трансформер остаётся на GPU во время декодирования, после чего память исчерпывается. При
40 ГиБ пиковое потребление составляет около 50 ГиБ на GPU объёмом 80 ГБ. Включите offload после <code>load_components</code>,
чтобы он применялся ко всем компонентам.</p> <!--[1--><h2 class="relative group"><a id="notes" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#notes"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Примечания<!----></span></h2><!--]--><!----> <ul><li><code>image</code> — это изображение в формате PIL, а <code>audio</code> — монофоническая волновая форма с плавающей точкой в диапазоне [-1, 1] при частоте дискретизации 16 кГц, представленная в виде тензора или массива с формой <code>(samples,)</code>; конвейер не читает файлы.</li> <li>Верхнеуровневые блоки (<code>text_encoder</code>, <code>vae_encoder</code>, <code>denoise</code>, <code>decode</code>) также могут выполняться независимо; при их отдельном запуске передавайте каждому одни и те же значения <code>num_latent_frames</code> и <code>fps</code>.</li> <li>Количество фрагментов определяется длиной аудиодорожки, если не передан параметр <code>num_chunks</code>. В любом случае видео обрезается по длине дорожки, поэтому фрагменты, выходящие за её пределы, вычисляются и отбрасываются с выводом предупреждения.</li> <li>При использовании guidance безусловный проход подставляет негативный промпт <em>и</em> заглушает речь, поэтому guidance является совместным для текста и речи.</li></ul> <!--[1--><h2 class="relative group"><a id="diffusers.KandinskyS2VModularPipeline" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#diffusers.KandinskyS2VModularPipeline"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->KandinskyS2VModularPipeline<!----></span></h2><!--]--><!----> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><div><span class="group flex space-x-1.5 items-center text-gray-800 bg-gradient-to-r rounded-tr-lg -mt-4 -ml-4 pt-3 px-2.5" id="diffusers.KandinskyS2VModularPipeline"><!----><h3 class="!m-0"><span class="flex-1 break-all md:text-lg bg-gradient-to-r px-2.5 py-1.5 rounded-xl from-indigo-50/70 to-white dark:from-gray-900 dark:to-gray-950 dark:text-indigo-300 text-indigo-700"><svg class="mr-1.5 text-indigo-500 inline-block -mt-0.5" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width=".8em" height=".8em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 24 24"><path class="uim-quaternary" d="M20.23 7.24L12 12L3.77 7.24a1.98 1.98 0 0 1 .7-.71L11 2.76c.62-.35 1.38-.35 2 0l6.53 3.77c.29.173.531.418.7.71z" opacity=".25" fill="currentColor"></path><path class="uim-tertiary" d="M12 12v9.5a2.09 2.09 0 0 1-.91-.21L4.5 17.48a2.003 2.003 0 0 1-1-1.73v-7.5a2.06 2.06 0 0 1 .27-1.01L12 12z" opacity=".5" fill="currentColor"></path><path class="uim-primary" d="M20.5 8.25v7.5a2.003 2.003 0 0 1-1 1.73l-6.62 3.82c-.275.13-.576.198-.88.2V12l8.23-4.76c.175.308.268.656.27 1.01z" fill="currentColor"></path></svg><span class="font-light">класс</span> <span class="font-medium">diffusers.</span><span class="font-semibold">KandinskyS2VModularPipeline</span></span></h3><!----> <a id="diffusers.KandinskyS2VModularPipeline" class="header-link invisible with-hover:group-hover:visible pr-2" href="#diffusers.KandinskyS2VModularPipeline"><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></a> <!--[0--><a class="!ml-auto !text-gray-400 !no-underline text-sm flex items-center" href="https://github.com/huggingface/diffusers/blob/main/src/diffusers/modular_pipelines/kandinsky_s2v/modular_pipeline.py#L91" target="_blank"><span>&lt;</span> <span class="hidden md:block mx-0.5 hover:!underline">source</span> <span>></span></a><!--]--></span> <!--[0--><p class="font-mono text-xs md:text-sm !leading-relaxed !my-6"><span>(</span> <!--[--><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">blocks<span class="opacity-60">: diffusers.modular_pipelines.modular_pipeline.ModularPipelineBlocks | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">pretrained_model_name_or_path<span class="opacity-60">: str | os.PathLike | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">components_manager<span class="opacity-60">: diffusers.modular_pipelines.components_manager.ComponentsManager | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">collection<span class="opacity-60">: str | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">workflow<span class="opacity-60">: str | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">modular_config_dict<span class="opacity-60">: dict[str, typing.Any] | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">config_dict<span class="opacity-60">: dict[str, typing.Any] | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">**kwargs<span class="opacity-60"></span></span></span><!--]--> <span>)</span> <!--[-1--><!--]--></p><!--]--> <div class="!mb-10 relative docstring-details "><!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--></div></div><!----> <p>Модульный ModularPipeline для генерации видео из речи в Kandinsky S2V 1.0.</p></div> <!--[1--><h2 class="relative group"><a id="diffusers.KandinskyS2VAutoBlocks" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#diffusers.KandinskyS2VAutoBlocks"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->KandinskyS2VAutoBlocks<!----></span></h2><!--]--><!----> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><div><span class="group flex space-x-1.5 items-center text-gray-800 bg-gradient-to-r rounded-tr-lg -mt-4 -ml-4 pt-3 px-2.5" id="diffusers.KandinskyS2VAutoBlocks"><!----><h3 class="!m-0"><span class="flex-1 break-all md:text-lg bg-gradient-to-r px-2.5 py-1.5 rounded-xl from-indigo-50/70 to-white dark:from-gray-900 dark:to-gray-950 dark:text-indigo-300 text-indigo-700"><svg class="mr-1.5 text-indigo-500 inline-block -mt-0.5" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width=".8em" height=".8em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 24 24"><path class="uim-quaternary" d="M20.23 7.24L12 12L3.77 7.24a1.98 1.98 0 0 1 .7-.71L11 2.76c.62-.35 1.38-.35 2 0l6.53 3.77c.29.173.531.418.7.71z" opacity=".25" fill="currentColor"></path><path class="uim-tertiary" d="M12 12v9.5a2.09 2.09 0 0 1-.91-.21L4.5 17.48a2.003 2.003 0 0 1-1-1.73v-7.5a2.06 2.06 0 0 1 .27-1.01L12 12z" opacity=".5" fill="currentColor"></path><path class="uim-primary" d="M20.5 8.25v7.5a2.003 2.003 0 0 1-1 1.73l-6.62 3.82c-.275.13-.576.198-.88.2V12l8.23-4.76c.175.308.268.656.27 1.01z" fill="currentColor"></path></svg><span class="font-light">класс</span> <span class="font-medium">diffusers.</span><span class="font-semibold">KandinskyS2VAutoBlocks</span></span></h3><!----> <a id="diffusers.KandinskyS2VAutoBlocks" class="header-link invisible with-hover:group-hover:visible pr-2" href="#diffusers.KandinskyS2VAutoBlocks"><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></a> <!--[0--><a class="!ml-auto !text-gray-400 !no-underline text-sm flex items-center" href="https://github.com/huggingface/diffusers/blob/main/src/diffusers/modular_pipelines/kandinsky_s2v/modular_blocks_kandinsky_s2v.py#L108" target="_blank"><span>&lt;</span> <span class="hidden md:block mx-0.5 hover:!underline">source</span> <span>></span></a><!--]--></span> <!--[0--><p class="font-mono text-xs md:text-sm !leading-relaxed !my-6"><span>(</span> <!--[--><!--]--> <span>)</span> <!--[-1--><!--]--></p><!--]--> <div class="!mb-10 relative docstring-details "><!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--></div></div><!----> <p>Модульный конвейер для генерации видео из речи с использованием Kandinsky S2V 1.0: видео аватара из опорного кадра и аудиодорожки, генерируемое по частям.</p> <p>Компоненты:
text_encoder (<code>Qwen2_5OmniThinkerForConditionalGeneration</code>) tokenizer (<code>Qwen2_5OmniProcessor</code>) text_encoder_2
(<code>CLIPTextModel</code>) tokenizer_2 (<code>CLIPTokenizer</code>) vae (<code>AutoencoderKLHunyuanVideo</code>) video_processor
(<code>VideoProcessor</code>) transformer (<code>KandinskyS2VTransformer3DModel</code>) scheduler
(<code>FlowMatchEulerDiscreteScheduler</code>) guider (<code>ClassifierFreeGuidance</code>)</p> <p>Inputs:
prompt (<code>str</code>, <em>optional</em>, defaults to The video presents a stable medium close-up of a subject facing the
camera directly. The subject exhibits natural, lifelike body dynamics: the head and shoulders move fluidly
with the rhythm of speech. The lighting is even, maintaining a balanced natural exposure to clearly define
facial features without washout. Throughout the sequence, the subject engages in continuous speech marked by
precise, high-fidelity articulation. The lips move in a rhythmic, natural pattern: alternating between brief
firm closures and relaxed openings. During these openings, the tips of the upper teeth are momentarily and
realistically visible, exhibiting natural enamel texture. The upper lip lifts subtly over the teeth during
enunciation, while the jaw performs minimal vertical oscillations synchronized with the speech cadence.
Facial micro-expressions are active yet restrained; eyebrows rise and lower slightly with minute asymmetry.
The eyes maintain a steady, engaged gaze forward, punctuated by smooth, deliberate blinks. ):
The prompt to guide the generation.
negative_prompt (<code>str</code>, <em>optional</em>):
The prompt for the unconditional pass of the guidance. Defaults to <code>DEFAULT_NEGATIVE_PROMPT</code> when the
guidance needs one.
audio (<code>Tensor | ndarray</code>):
The speech track that drives the generation: a mono float waveform in [-1, 1] at 16000 Hz, shaped
(samples,).
num_latent_frames (<code>int</code>, <em>optional</em>, defaults to 32):
Latent frames per chunk; 32 gives 125 frames (5 s at 25 fps).
fps (<code>int</code>, <em>optional</em>, defaults to 25):
Frame rate the chunk length is computed at.
num_chunks (<code>int</code>, <em>optional</em>):
Number of chunks to generate. Defaults to however many cover the speech track; the video is trimmed to
the track either way.
image (<code>Image</code>):
The reference frame to animate.
height (<code>int</code>, <em>optional</em>):
The height in pixels of the generated video. Defaults to the 512 bucket closest to the reference frame.
width (<code>int</code>, <em>optional</em>):
The width in pixels of the generated video, chosen together with <code>height</code>.
generator (<code>Generator</code>, <em>optional</em>):
Torch generator for deterministic generation.
num_inference_steps (<code>int</code>, <em>optional</em>, defaults to 50):
The number of denoising steps.
scale_factor (<code>tuple</code>, <em>optional</em>, defaults to (1.0, 2.0, 2.0)):
Per-axis RoPE scale for the resolution bucket.
output_type (<code>str</code>, <em>optional</em>, defaults to pil):
Output format: ‘pil’, ‘np’, ‘pt’.</p> <p>Вывод:
видео (<code>list</code>):
Сгенерированные видео.</p></div> <!--[1--><h2 class="relative group"><a id="diffusers.KandinskyS2VDistilledModularPipeline" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#diffusers.KandinskyS2VDistilledModularPipeline"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->KandinskyS2VDistilledModularPipeline<!----></span></h2><!--]--><!----> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><div><span class="group flex space-x-1.5 items-center text-gray-800 bg-gradient-to-r rounded-tr-lg -mt-4 -ml-4 pt-3 px-2.5" id="diffusers.KandinskyS2VDistilledModularPipeline"><!----><h3 class="!m-0"><span class="flex-1 break-all md:text-lg bg-gradient-to-r px-2.5 py-1.5 rounded-xl from-indigo-50/70 to-white dark:from-gray-900 dark:to-gray-950 dark:text-indigo-300 text-indigo-700"><svg class="mr-1.5 text-indigo-500 inline-block -mt-0.5" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width=".8em" height=".8em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 24 24"><path class="uim-quaternary" d="M20.23 7.24L12 12L3.77 7.24a1.98 1.98 0 0 1 .7-.71L11 2.76c.62-.35 1.38-.35 2 0l6.53 3.77c.29.173.531.418.7.71z" opacity=".25" fill="currentColor"></path><path class="uim-tertiary" d="M12 12v9.5a2.09 2.09 0 0 1-.91-.21L4.5 17.48a2.003 2.003 0 0 1-1-1.73v-7.5a2.06 2.06 0 0 1 .27-1.01L12 12z" opacity=".5" fill="currentColor"></path><path class="uim-primary" d="M20.5 8.25v7.5a2.003 2.003 0 0 1-1 1.73l-6.62 3.82c-.275.13-.576.198-.88.2V12l8.23-4.76c.175.308.268.656.27 1.01z" fill="currentColor"></path></svg><span class="font-light">класс</span> <span class="font-medium">diffusers.</span><span class="font-semibold">KandinskyS2VDistilledModularPipeline</span></span></h3><!----> <a id="diffusers.KandinskyS2VDistilledModularPipeline" class="header-link invisible with-hover:group-hover:visible pr-2" href="#diffusers.KandinskyS2VDistilledModularPipeline"><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></a> <!--[0--><a class="!ml-auto !text-gray-400 !no-underline text-sm flex items-center" href="https://github.com/huggingface/diffusers/blob/main/src/diffusers/modular_pipelines/kandinsky_s2v/modular_pipeline.py#L109" target="_blank"><span>&lt;</span> <span class="hidden md:block mx-0.5 hover:!underline">source</span> <span>></span></a><!--]--></span> <!--[0--><p class="font-mono text-xs md:text-sm !leading-relaxed !my-6"><span>(</span> <!--[--><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">blocks<span class="opacity-60">: diffusers.modular_pipelines.modular_pipeline.ModularPipelineBlocks | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">pretrained_model_name_or_path<span class="opacity-60">: str | os.PathLike | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">components_manager<span class="opacity-60">: diffusers.modular_pipelines.components_manager.ComponentsManager | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">collection<span class="opacity-60">: str | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">workflow<span class="opacity-60">: str | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">modular_config_dict<span class="opacity-60">: dict[str, typing.Any] | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">config_dict<span class="opacity-60">: dict[str, typing.Any] | None = None</span></span></span><span class="comma cursor-default"><span class="rounded hover:bg-black hover:text-white dark:hover:bg-white dark:hover:text-black">**kwargs<span class="opacity-60"></span></span></span><!--]--> <span>)</span> <!--[-1--><!--]--></p><!--]--> <div class="!mb-10 relative docstring-details "><!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--></div></div><!----> <p>ModularPipeline для дистиллированных контрольных точек Kandinsky S2V 1.0 от pi-Flow.</p> <p>Guidance дистиллируется в ученическую модель, поэтому её блоки не содержат guider и не используют отрицательный промпт, а на каждый сегмент выполняется один прямой проход.</p></div> <!--[1--><h2 class="relative group"><a id="diffusers.KandinskyS2VDistilledAutoBlocks" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#diffusers.KandinskyS2VDistilledAutoBlocks"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->KandinskyS2VDistilledAutoBlocks<!----></span></h2><!--]--><!----> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><div><span class="group flex space-x-1.5 items-center text-gray-800 bg-gradient-to-r rounded-tr-lg -mt-4 -ml-4 pt-3 px-2.5" id="diffusers.KandinskyS2VDistilledAutoBlocks"><!----><h3 class="!m-0"><span class="flex-1 break-all md:text-lg bg-gradient-to-r px-2.5 py-1.5 rounded-xl from-indigo-50/70 to-white dark:from-gray-900 dark:to-gray-950 dark:text-indigo-300 text-indigo-700"><svg class="mr-1.5 text-indigo-500 inline-block -mt-0.5" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width=".8em" height=".8em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 24 24"><path class="uim-quaternary" d="M20.23 7.24L12 12L3.77 7.24a1.98 1.98 0 0 1 .7-.71L11 2.76c.62-.35 1.38-.35 2 0l6.53 3.77c.29.173.531.418.7.71z" opacity=".25" fill="currentColor"></path><path class="uim-tertiary" d="M12 12v9.5a2.09 2.09 0 0 1-.91-.21L4.5 17.48a2.003 2.003 0 0 1-1-1.73v-7.5a2.06 2.06 0 0 1 .27-1.01L12 12z" opacity=".5" fill="currentColor"></path><path class="uim-primary" d="M20.5 8.25v7.5a2.003 2.003 0 0 1-1 1.73l-6.62 3.82c-.275.13-.576.198-.88.2V12l8.23-4.76c.175.308.268.656.27 1.01z" fill="currentColor"></path></svg><span class="font-light">класс</span> <span class="font-medium">diffusers.</span><span class="font-semibold">KandinskyS2VDistilledAutoBlocks</span></span></h3><!----> <a id="diffusers.KandinskyS2VDistilledAutoBlocks" class="header-link invisible with-hover:group-hover:visible pr-2" href="#diffusers.KandinskyS2VDistilledAutoBlocks"><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></a> <!--[0--><a class="!ml-auto !text-gray-400 !no-underline text-sm flex items-center" href="https://github.com/huggingface/diffusers/blob/main/src/diffusers/modular_pipelines/kandinsky_s2v/modular_blocks_kandinsky_s2v_distilled.py#L103" target="_blank"><span>&lt;</span> <span class="hidden md:block mx-0.5 hover:!underline">source</span> <span>></span></a><!--]--></span> <!--[0--><p class="font-mono text-xs md:text-sm !leading-relaxed !my-6"><span>(</span> <!--[--><!--]--> <span>)</span> <!--[-1--><!--]--></p><!--]--> <div class="!mb-10 relative docstring-details "><!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--></div></div><!----> <p>Модульный конвейер для генерации видео из речи с использованием дистиллированных чекпоинтов pi-Flow Kandinsky S2V 1.0: 16 сегментов на чанк и без гайдинга, что дистиллируется в ученическую модель.</p> <p>Компоненты:
text_encoder (<code>Qwen2_5OmniThinkerForConditionalGeneration</code>) tokenizer (<code>Qwen2_5OmniProcessor</code>) text_encoder_2
(<code>CLIPTextModel</code>) tokenizer_2 (<code>CLIPTokenizer</code>) vae (<code>AutoencoderKLHunyuanVideo</code>) video_processor
(<code>VideoProcessor</code>) transformer (<code>KandinskyS2VTransformer3DModel</code>) scheduler (<code>PiflowScheduler</code>)</p> <p>Inputs:
prompt (<code>str</code>, <em>optional</em>, defaults to The video presents a stable medium close-up of a subject facing the
camera directly. The subject exhibits natural, lifelike body dynamics: the head and shoulders move fluidly
with the rhythm of speech. The lighting is even, maintaining a balanced natural exposure to clearly define
facial features without washout. Throughout the sequence, the subject engages in continuous speech marked by
precise, high-fidelity articulation. The lips move in a rhythmic, natural pattern: alternating between brief
firm closures and relaxed openings. During these openings, the tips of the upper teeth are momentarily and
realistically visible, exhibiting natural enamel texture. The upper lip lifts subtly over the teeth during
enunciation, while the jaw performs minimal vertical oscillations synchronized with the speech cadence.
Facial micro-expressions are active yet restrained; eyebrows rise and lower slightly with minute asymmetry.
The eyes maintain a steady, engaged gaze forward, punctuated by smooth, deliberate blinks. ):
The prompt to guide the generation.
audio (<code>Tensor | ndarray</code>):
The speech track that drives the generation: a mono float waveform in [-1, 1] at 16000 Hz, shaped
(samples,).
num_latent_frames (<code>int</code>, <em>optional</em>, defaults to 32):
Latent frames per chunk; 32 gives 125 frames (5 s at 25 fps).
fps (<code>int</code>, <em>optional</em>, defaults to 25):
Frame rate the chunk length is computed at.
num_chunks (<code>int</code>, <em>optional</em>):
Number of chunks to generate. Defaults to however many cover the speech track; the video is trimmed to
the track either way.
image (<code>Image</code>):
The reference frame to animate.
height (<code>int</code>, <em>optional</em>):
The height in pixels of the generated video. Defaults to the 512 bucket closest to the reference frame.
width (<code>int</code>, <em>optional</em>):
The width in pixels of the generated video, chosen together with <code>height</code>.
generator (<code>Generator</code>, <em>optional</em>):
Torch generator for deterministic generation.
num_inference_steps (<code>int</code>, <em>optional</em>, defaults to 16):
The number of denoising steps.
scale_factor (<code>tuple</code>, <em>optional</em>, defaults to (1.0, 2.0, 2.0)):
Per-axis RoPE scale for the resolution bucket.
output_type (<code>str</code>, <em>optional</em>, defaults to pil):
Output format: ‘pil’, ‘np’, ‘pt’.</p> <p>Вывод:
видео (<code>list</code>):
Сгенерированные видео.</p></div> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/diffusers/blob/main/docs/source/en/api/pipelines/kandinsky_s2v.md" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg><!----> <span><span class="underline">Update</span> on GitHub</span></a><!----> <p></p><!--]--><!----><!--]--><!--]--><!--]--> <!--[-1--><!--]--><!--]-->
			
			<script>
				{
					__sveltekit_g37pnb = {
						base: "/docs/diffusers/main/en",
						assets: "/docs/diffusers/main/en"
					};
					const element = document.currentScript.parentElement;
					Promise.all([
						import("/docs/diffusers/main/en/_app/immutable/entry/start.D3LnKZ4R.js"),
						import("/docs/diffusers/main/en/_app/immutable/entry/app.C0rBoMxR.js")
					]).then(([kit, app]) => {
						kit.start(app, element, {
							node_ids: [0, 197],
							data: [null,null],
							form: null,
							error: null
						});
					});
				}
			</script>
		
Xet Storage Details

Size:: 52.2 kB
Xet hash:: 62bf53b8d4dd3ccc0e3b484ef464b0d5697a36487e3385291293ab58e23f78aa
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.