Buckets:
| import"../chunks/DsnmJJEf.js";import{i as k,h as T,C as b,H as c,D as l,E as D,s as S}from"../chunks/DBNuz8hP.js";import{p as M,o as w,s as e,f as K,a as p,b as V,c as f,d as h,n as _,r as u}from"../chunks/CQ2f15a1.js";const P='{"title":"KandinskyS2VTransformer3DModel","local":"kandinskys2vtransformer3dmodel","sections":[{"title":"KandinskyS2VTransformer3DModel","local":"diffusers.KandinskyS2VTransformer3DModel","sections":[],"depth":2}],"depth":1}';var E=h('<meta name="hf:doc:metadata"/>'),j=h(`<p></p> <!> <!> <p>The Diffusion Transformer (DiT) of Kandinsky S2V 1.0, conditioned on a reference | |
| frame and a speech track.</p> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Speech-to-video diffusion transformer.</p> <p>Parameter names match the training checkpoints, so a converted <em>.safetensors</em> loads with no key remapping. | |
| Inference is batch size 1: the sequence is packed and the speech cross-attention is frame-aligned, so <em>forward</em> takes a leading batch axis of 1 and squeezes it.</p> <p>The visual stream is one packed sequence <code>[memory | ref | visual]</code>:</p> <ul><li><code>memory</code> — FramePack context from the previous chunk at 4x / 2x / 1x temporal downsampling, placed at negative | |
| temporal RoPE positions;</li> <li><code>ref</code> — the reference frame, pinned to temporal RoPE index <code>index_rope_ref_image</code>;</li> <li><code>visual</code> — the noisy latent being denoised.</li></ul> <p><code>memory</code> and <code>ref</code> are clean, so they are modulated with <code>t = 0</code> while <code>visual</code> uses the real timestep; the | |
| modulation holds both rows and an index selects per token.</p> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Denoise one chunk.</p> <p>Batch size must be 1: the sequence is packed and the speech cross-attention is frame-aligned.</p></div></div> <!> <p></p>`,1);function q(v,g){M(g,!1),w(()=>{new URLSearchParams(window.location.search).get("fw")}),k();var r=j();T("1qtreh1",d=>{var m=E();S(m,"content",P),p(d,m)});var n=e(K(r),2);b(n,{containerStyle:"float: inline-end; margin-inline-start: 10px; display: inline-flex; position: relative; z-index: 10;"});var s=e(n,2);c(s,{title:"KandinskyS2VTransformer3DModel",local:"kandinskys2vtransformer3dmodel",headingTag:"h1"});var a=e(s,4);c(a,{title:"KandinskyS2VTransformer3DModel",local:"diffusers.KandinskyS2VTransformer3DModel",headingTag:"h2"});var o=e(a,2),t=f(o);l(t,{name:"class diffusers.KandinskyS2VTransformer3DModel",anchor:"diffusers.KandinskyS2VTransformer3DModel",source:"https://github.com/huggingface/diffusers/blob/vr_15249/src/diffusers/models/transformers/transformer_kandinsky_s2v.py#L487",parameters:[{name:"in_visual_dim",val:": int = 16"},{name:"out_visual_dim",val:": int = 16"},{name:"in_text_dim",val:": int = 3584"},{name:"in_text_dim2",val:": int = 768"},{name:"in_audio_dim",val:": int = 3584"},{name:"time_dim",val:": int = 512"},{name:"patch_size",val:": tuple[int, int, int] = (1, 2, 2)"},{name:"frame_buckets",val:": tuple[int, int, int] = (2, 4, 24)"},{name:"enable_framepack",val:": bool = True"},{name:"index_rope_ref_image",val:": int = 40"},{name:"patch_proj1x_coef",val:": tuple[int, int, int] = (1, 1, 1)"},{name:"patch_proj2x_coef",val:": tuple[int, int, int] = (2, 2, 2)"},{name:"patch_proj4x_coef",val:": tuple[int, int, int] = (4, 4, 4)"},{name:"model_dim",val:": int = 1792"},{name:"ff_dim",val:": int = 7168"},{name:"num_text_blocks",val:": int = 2"},{name:"num_audio_blocks",val:": int = 2"},{name:"num_visual_blocks",val:": int = 32"},{name:"axes_dims",val:": tuple[int, int, int] = (16, 24, 24)"},{name:"audio_inject_layers",val:": tuple[int, ...] = ()"},{name:"text_rope_max_pos",val:": int = 4096"}]});var i=e(t,12),y=f(i);l(y,{name:"forward",anchor:"diffusers.KandinskyS2VTransformer3DModel.forward",source:"https://github.com/huggingface/diffusers/blob/vr_15249/src/diffusers/models/transformers/transformer_kandinsky_s2v.py#L632",parameters:[{name:"hidden_states",val:": Tensor"},{name:"encoder_hidden_states",val:": Tensor"},{name:"audio_hidden_states",val:": Tensor"},{name:"reference_hidden_states",val:": Tensor"},{name:"memory_hidden_states",val:": tuple[Tensor, Tensor, Tensor]"},{name:"timestep",val:": Tensor"},{name:"pooled_projections",val:": Tensor"},{name:"visual_rope_pos",val:": list[Tensor]"},{name:"reference_rope_pos",val:": list[Tensor]"},{name:"memory_rope_pos",val:": tuple[list[Tensor], list[Tensor], list[Tensor]]"},{name:"text_rope_pos",val:": Tensor"},{name:"audio_rope_pos",val:": Tensor"},{name:"scale_factor",val:": tuple[float, float, float] = (1.0, 1.0, 1.0)"},{name:"return_dict",val:": bool = True"}],parametersDescription:[{anchor:"diffusers.KandinskyS2VTransformer3DModel.forward.hidden_states",description:"<strong>hidden_states</strong> — noisy latent, <code>(1, T, H, W, C)</code>.",name:"hidden_states"},{anchor:"diffusers.KandinskyS2VTransformer3DModel.forward.encoder_hidden_states",description:"<strong>encoder_hidden_states</strong> — text embeddings, <code>(1, S_text, in_text_dim)</code>.",name:"encoder_hidden_states"},{anchor:"diffusers.KandinskyS2VTransformer3DModel.forward.audio_hidden_states",description:"<strong>audio_hidden_states</strong> — speech features, <code>(1, S_audio, in_audio_dim)</code>.",name:"audio_hidden_states"},{anchor:"diffusers.KandinskyS2VTransformer3DModel.forward.reference_hidden_states",description:"<strong>reference_hidden_states</strong> — reference frame latent, <code>(1, 1, H, W, C)</code>.",name:"reference_hidden_states"},{anchor:"diffusers.KandinskyS2VTransformer3DModel.forward.memory_hidden_states",description:"<strong>memory_hidden_states</strong> — FramePack memory at 4x / 2x / 1x, each <code>(1, T_i, H_i, W_i, C)</code>.",name:"memory_hidden_states"},{anchor:"diffusers.KandinskyS2VTransformer3DModel.forward.timestep",description:"<strong>timestep</strong> — current timestep, <code>(1,)</code>.",name:"timestep"},{anchor:"diffusers.KandinskyS2VTransformer3DModel.forward.pooled_projections",description:"<strong>pooled_projections</strong> — pooled CLIP text embedding, <code>(1, in_text_dim2)</code>.",name:"pooled_projections"},{anchor:"diffusers.KandinskyS2VTransformer3DModel.forward.visual_rope_pos",description:"<strong>visual_rope_pos</strong> — per-axis RoPE positions of the noisy latent.",name:"visual_rope_pos"},{anchor:"diffusers.KandinskyS2VTransformer3DModel.forward.reference_rope_pos",description:"<strong>reference_rope_pos</strong> — per-axis RoPE positions of the reference frame.",name:"reference_rope_pos"},{anchor:"diffusers.KandinskyS2VTransformer3DModel.forward.memory_rope_pos",description:"<strong>memory_rope_pos</strong> — per-axis RoPE positions of each FramePack memory scale.",name:"memory_rope_pos"},{anchor:"diffusers.KandinskyS2VTransformer3DModel.forward.text_rope_pos",description:"<strong>text_rope_pos</strong> — 1-D RoPE positions of the text tokens.",name:"text_rope_pos"},{anchor:"diffusers.KandinskyS2VTransformer3DModel.forward.audio_rope_pos",description:"<strong>audio_rope_pos</strong> — latent frame each speech token belongs to.",name:"audio_rope_pos"},{anchor:"diffusers.KandinskyS2VTransformer3DModel.forward.scale_factor",description:"<strong>scale_factor</strong> — RoPE scale per axis, set by the resolution bucket.",name:"scale_factor"},{anchor:"diffusers.KandinskyS2VTransformer3DModel.forward.return_dict",description:"<strong>return_dict</strong> — return a <code>Transformer2DModelOutput</code> instead of a tuple.",name:"return_dict"}],returnDescription:` | |
| <p>the predicted velocity, shaped like | |
| <code>hidden_states</code>.</p> | |
| `,returnType:` | |
| <p>[<em>~models.modeling_outputs.Transformer2DModelOutput</em>] or <em>tuple</em></p> | |
| `}),_(4),u(i),u(o);var x=e(o,2);D(x,{source:"https://github.com/huggingface/diffusers/blob/main/docs/source/en/api/models/kandinsky_s2v_transformer.md"}),_(2),p(v,r),V()}export{q as component}; | |
Xet Storage Details
- Size:
- 8.09 kB
- Xet hash:
- 23ab04aa05589e2641e4d26a4c9f1f7c665839ddd990cde7fa979748d1ae9500
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.