MiniMax H3

Add the ComfyUI H3 corrector node, including a tunable apply node. The stock Apply node is unchanged and still loads the corrector with H3. Apply H3 Corrector (Tunable) can unload H3 for the corrector forward so a 32 GB card that already runs H3 can run the corrector without holding both networks at once.

#6
by sandpies - opened
comfy/ComfyUI-H3-Corrector/README.md ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ # ComfyUI H3 Corrector
2
+
3
+ Copy this folder into ComfyUI/custom_nodes/ and restart. Put corrector files in models/h3_correctors/.
4
+
5
+ - **Load H3 Corrector** reads the safetensors in fp32.
6
+ - **Apply H3 Corrector** is the stock node. It keeps the corrector loaded with H3 for the whole sample.
7
+ - **Apply H3 Corrector (Tunable)** adds a step window, strength ramp, per-frame cap, virtual step size, and **Swap Models**. Swap unloads H3, runs the corrector, then reloads H3. Leave it on unless the card has room for both.
8
+
9
+ On a 20-step sample the window that held was steps 0-8, strength 1.0, budget 0.5. On an 8-step turbo schedule the matching early window was steps 0-3, with virtual_steps 20. Budget 1.0 and strength 1.5 pushed the sample onto a different scene.
comfy/ComfyUI-H3-Corrector/__init__.py ADDED
@@ -0,0 +1,248 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """MiniMax H3 corrector for ComfyUI.
2
+
3
+ The corrector is a separate 2B network trained against one H3 checkpoint (S3). At each denoising step it reads the
4
+ noisy video latent, H3's guided video velocity, the step's sigma interval and the raw Qwen3-VL caption states, and
5
+ predicts a residual. The residual is capped at a fraction of the velocity's RMS and added to the video velocity;
6
+ audio is left as predicted.
7
+
8
+ Apply H3 Corrector (Tunable) is the same residual with extra knobs. Swap Models unloads H3 for the corrector forward so the two networks are not resident together. The stock Apply node is unchanged.
9
+
10
+ Load H3 Corrector -> Apply H3 Corrector (model in, model out) -> any sampler
11
+
12
+ Correctors are single .safetensors files in models/h3_correctors/, with the architecture in the file's metadata.
13
+ """
14
+ from __future__ import annotations
15
+
16
+ import logging
17
+ import time
18
+ import os
19
+
20
+ import torch
21
+ from safetensors import safe_open
22
+ from safetensors.torch import load_file
23
+
24
+ import comfy.model_management
25
+ import comfy.model_patcher
26
+ import comfy.utils
27
+ import folder_paths
28
+
29
+ from .corrector import CorrectorConfig, H3Corrector, bound_residual
30
+
31
+ log = logging.getLogger("H3Corrector")
32
+ CORRECTOR_DIR = os.path.join(folder_paths.models_dir, "h3_correctors")
33
+ os.makedirs(CORRECTOR_DIR, exist_ok=True)
34
+ if "h3_correctors" in folder_paths.folder_names_and_paths:
35
+ folder_paths.add_model_folder_path("h3_correctors", CORRECTOR_DIR)
36
+ else:
37
+ folder_paths.folder_names_and_paths["h3_correctors"] = ([CORRECTOR_DIR], {".safetensors"})
38
+
39
+
40
+ class LoadH3Corrector:
41
+ @classmethod
42
+ def INPUT_TYPES(cls):
43
+ return {"required": {"corrector": (folder_paths.get_filename_list("h3_correctors"),
44
+ {"tooltip": "A corrector .safetensors in models/h3_correctors."})}}
45
+
46
+ RETURN_TYPES = ("H3_CORRECTOR",)
47
+ FUNCTION = "load"
48
+ CATEGORY = "MiniMax H3/corrector"
49
+ DESCRIPTION = "Loads an H3 corrector in fp32 (bf16 weights change its output several-fold)."
50
+
51
+ def load(self, corrector):
52
+ path = folder_paths.get_full_path_or_raise("h3_correctors", corrector)
53
+ with safe_open(path, "pt") as handle:
54
+ config = CorrectorConfig.from_metadata(handle.metadata())
55
+ network = H3Corrector(config)
56
+ network.load_state_dict(load_file(path, device="cpu"), strict=True)
57
+ network.eval().requires_grad_(False)
58
+ patcher = comfy.model_patcher.CoreModelPatcher(
59
+ network, load_device=comfy.model_management.get_torch_device(),
60
+ offload_device=comfy.model_management.unet_offload_device())
61
+ return ({"patcher": patcher, "network": network, "name": os.path.splitext(corrector)[0]},)
62
+
63
+
64
+ def _step_interval(sigma, sample_sigmas):
65
+ """(step index, sigma_next) for the schedule interval containing sigma; (None, None) if there is no schedule."""
66
+ if sample_sigmas is None:
67
+ return None, None
68
+ sigmas = sample_sigmas.to(device="cpu", dtype=torch.float64).flatten()
69
+ value = float(sigma.flatten()[0])
70
+ # sigmas fall monotonically; an extra evaluation between two points (Heun etc.) belongs to the interval it sits in
71
+ index = min(max(int((sigmas[:-1] >= value - 1e-6).sum()) - 1, 0), len(sigmas) - 2)
72
+ return index, float(sigmas[index + 1])
73
+
74
+
75
+ class ApplyH3Corrector:
76
+ @classmethod
77
+ def INPUT_TYPES(cls):
78
+ return {
79
+ "required": {
80
+ "model": ("MODEL",),
81
+ "corrector": ("H3_CORRECTOR",),
82
+ "strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 2.0, "step": 0.05,
83
+ "tooltip": "Multiplier on the capped residual. 1 = as trained."}),
84
+ "budget_fraction": ("FLOAT", {"default": 0.5, "min": 0.05, "max": 1.0, "step": 0.05,
85
+ "tooltip": "Cap on the residual as a fraction of the velocity RMS. The S3 corrector was trained at 0.5."}),
86
+ "velocity_floor": ("FLOAT", {"default": 0.001, "min": 0.0001, "max": 1.0, "step": 0.0001,
87
+ "tooltip": "Lower bound on the velocity RMS used for the cap."}),
88
+ "start_step": ("INT", {"default": 0, "min": 0, "max": 1000,
89
+ "tooltip": "First step (0-based) the corrector is active."}),
90
+ "end_step": ("INT", {"default": -1, "min": -1, "max": 1000,
91
+ "tooltip": "Last step (0-based, inclusive) the corrector is active. -1 = through the final step."}),
92
+ }
93
+ }
94
+
95
+ RETURN_TYPES = ("MODEL",)
96
+ FUNCTION = "apply"
97
+ CATEGORY = "MiniMax H3/corrector"
98
+ DESCRIPTION = ("Adds the corrector's capped residual to H3's guided video velocity on steps start_step..end_step "
99
+ "(all steps by default). Use with the H3 checkpoint the corrector was trained on (S3).")
100
+
101
+ def apply(self, model, corrector, strength, budget_fraction, velocity_floor, start_step, end_step):
102
+ network = corrector["network"]
103
+ name = corrector["name"]
104
+ last = float("inf") if end_step < 0 else end_step
105
+
106
+ def post_cfg(args):
107
+ denoised, x, sigma = args["denoised"], args["input"], args["sigma"]
108
+ options = args["model_options"].get("transformer_options", {})
109
+ index, sigma_next = _step_interval(sigma, options.get("sample_sigmas"))
110
+ if index is None:
111
+ log.warning("H3 corrector: no sample_sigmas in transformer_options, skipping")
112
+ return denoised
113
+ if strength == 0 or not start_step <= index <= last:
114
+ return denoised
115
+ shapes = getattr(args["model"], "latent_shapes", None)
116
+ if not shapes or len(shapes) < 2:
117
+ raise RuntimeError("H3 corrector expects the packed MiniMax H3 audio+video latent")
118
+ text = next((c.get("cross_attn") for c in (args.get("cond") or []) if c.get("cross_attn") is not None), None)
119
+ if text is None or text.shape[-1] != network.config.text_dim:
120
+ raise RuntimeError(f"H3 corrector needs the raw [1, L, {network.config.text_dim}] caption states (cross_attn)")
121
+ video_x, *_ = comfy.utils.unpack_latents(x, shapes)
122
+ video_d, *_ = comfy.utils.unpack_latents(denoised, shapes)
123
+ device = next(network.parameters()).device
124
+ s = float(sigma.flatten()[0])
125
+ z = video_x.to(device, torch.float32)
126
+ # ComfyUI's model output is v = noise - x0; the corrector works on H3's data-ward velocity (x0 - z) / sigma
127
+ velocity = (video_d.to(device, torch.float32) - z) / s
128
+ with torch.no_grad(), torch.autocast(device.type, dtype=torch.bfloat16, enabled=device.type == "cuda"):
129
+ raw = network(z, velocity, s, sigma_next, text.to(device, torch.float32))
130
+ applied, ratio = bound_residual(raw, velocity, budget_fraction, velocity_floor)
131
+ log.info("H3 corrector %s step %d: sigma %.4f -> %.4f, residual/velocity RMS %.3f",
132
+ name, index, s, sigma_next, float(ratio.mean()))
133
+ corrected = video_d.to(device, torch.float32) + s * strength * applied
134
+ flat = corrected.to(denoised.dtype).to(denoised.device).reshape(denoised.shape[0], 1, -1)
135
+ return torch.cat((flat, denoised[:, :, flat.shape[-1]:]), dim=-1)
136
+
137
+ patched = model.clone()
138
+ # ComfyUI then loads the corrector with the model and counts its memory when deciding what H3 offloads
139
+ patched.set_additional_models("h3_corrector", [corrector["patcher"]])
140
+ patched.set_model_sampler_post_cfg_function(post_cfg)
141
+ return (patched,)
142
+
143
+
144
+ def _cap(raw, velocity, budget_fraction, velocity_floor, per_frame, eps=1e-12):
145
+ raw, velocity = raw.float(), velocity.float()
146
+ dims = (1, 3, 4) if per_frame else tuple(range(1, raw.ndim)) # [B,C,T,H,W]: per frame keeps T
147
+ base = velocity.square().mean(dim=dims, keepdim=True).sqrt()
148
+ budget = budget_fraction * torch.clamp_min(base, velocity_floor)
149
+ scale = torch.rsqrt(1.0 + raw.square().mean(dim=dims, keepdim=True) / budget.square().clamp_min(eps))
150
+ applied = raw * scale
151
+ ratio = applied.square().mean().sqrt() / velocity.square().mean().sqrt().clamp_min(eps)
152
+ return applied, ratio
153
+
154
+
155
+ class ApplyH3CorrectorTunable:
156
+ @classmethod
157
+ def INPUT_TYPES(cls):
158
+ return {"required": {
159
+ "model": ("MODEL",), "corrector": ("H3_CORRECTOR",),
160
+ "strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 2.0, "step": 0.05}),
161
+ "budget_fraction": ("FLOAT", {"default": 0.5, "min": 0.05, "max": 1.0, "step": 0.05}),
162
+ "velocity_floor": ("FLOAT", {"default": 0.001, "min": 0.0001, "max": 1.0, "step": 0.0001}),
163
+ "start_step": ("INT", {"default": 0, "min": 0, "max": 1000}),
164
+ "end_step": ("INT", {"default": -1, "min": -1, "max": 1000}),
165
+ "virtual_steps": ("INT", {"default": 0, "min": 0, "max": 200, "tooltip": "0 = off. Tell the corrector the step is this-many-step sized."}),
166
+ "strength_end": ("FLOAT", {"default": -1.0, "min": -1.0, "max": 2.0, "step": 0.05, "tooltip": "-1 = off. Linear ramp to this by the last active step."}),
167
+ "fp32_forward": ("BOOLEAN", {"default": False}),
168
+ "per_frame_cap": ("BOOLEAN", {"default": False}),
169
+ "swap_models": ("BOOLEAN", {"default": True, "tooltip": "Unload H3, run the corrector, reload H3. One model on the GPU at a time."}),
170
+ }}
171
+
172
+ RETURN_TYPES = ("MODEL",)
173
+ FUNCTION = "apply"
174
+ CATEGORY = "MiniMax H3/corrector"
175
+
176
+ def apply(self, model, corrector, strength, budget_fraction, velocity_floor, start_step, end_step,
177
+ virtual_steps, strength_end, fp32_forward, per_frame_cap, swap_models):
178
+ network, name = corrector["network"], corrector["name"]
179
+ corr_patcher = corrector["patcher"]
180
+
181
+ def _exchange(target, unload, *, corrector):
182
+ t = time.perf_counter()
183
+ comfy.model_management.unload_model_and_clones(unload)
184
+ comfy.model_management.soft_empty_cache()
185
+ if corrector:
186
+ # 4 GB headroom for the forward. H3 is already unloaded, so this does not keep both resident.
187
+ comfy.model_management.load_models_gpu([target], memory_required=4 * 1024 ** 3, force_full_load=True)
188
+ else:
189
+ comfy.model_management.load_models_gpu([target])
190
+ return time.perf_counter() - t
191
+
192
+ def post_cfg(args):
193
+ denoised, x, sigma = args["denoised"], args["input"], args["sigma"]
194
+ sample_sigmas = args["model_options"].get("transformer_options", {}).get("sample_sigmas")
195
+ if sample_sigmas is None:
196
+ log.warning("no sample_sigmas, skipping"); return denoised
197
+ index, sigma_next, n = _interval(sigma, sample_sigmas)
198
+ last = n - 1 if end_step < 0 else min(end_step, n - 1)
199
+ if not start_step <= index <= last:
200
+ return denoised
201
+ k = strength
202
+ if strength_end >= 0 and last > start_step:
203
+ k = strength + (strength_end - strength) * (index - start_step) / (last - start_step)
204
+ if k == 0:
205
+ return denoised
206
+ shapes = getattr(args["model"], "latent_shapes", None)
207
+ if not shapes or len(shapes) < 2:
208
+ raise RuntimeError("expects the packed MiniMax H3 audio+video latent")
209
+ text = next((c.get("cross_attn") for c in (args.get("cond") or []) if c.get("cross_attn") is not None), None)
210
+ if text is None or text.shape[-1] != network.config.text_dim:
211
+ raise RuntimeError("needs raw caption states (cross_attn)")
212
+ video_x, *_ = comfy.utils.unpack_latents(x, shapes)
213
+ video_d, *_ = comfy.utils.unpack_latents(denoised, shapes)
214
+ s = float(sigma.flatten()[0])
215
+ told_next = sigma_next
216
+ if virtual_steps > 0:
217
+ told_next = max(s - (s - sigma_next) * n / virtual_steps, 0.0)
218
+ swapped = False
219
+ swap_in = 0.0
220
+ try:
221
+ if swap_models:
222
+ swap_in = _exchange(corr_patcher, patched, corrector=True)
223
+ swapped = True
224
+ device = comfy.model_management.get_torch_device()
225
+ z = video_x.to(device, torch.float32)
226
+ velocity = (video_d.to(device, torch.float32) - z) / s
227
+ use_bf16 = device.type == "cuda" and not fp32_forward
228
+ with torch.no_grad(), torch.autocast(device.type, dtype=torch.bfloat16, enabled=use_bf16):
229
+ raw = network(z, velocity, s, told_next, text.to(device, torch.float32))
230
+ applied, ratio = _cap(raw, velocity, budget_fraction, velocity_floor, per_frame_cap)
231
+ log.info("H3 corrector(T) %s step %d/%d: sigma %.4f -> %.4f (told %.4f), k %.3f, rms %.3f, swap_in %.2fs",
232
+ name, index, n, s, sigma_next, told_next, k, float(ratio), swap_in)
233
+ corrected = video_d.to(device, torch.float32) + s * k * applied
234
+ flat = corrected.to(denoised.dtype).to(denoised.device).reshape(denoised.shape[0], 1, -1)
235
+ return torch.cat((flat, denoised[:, :, flat.shape[-1]:]), dim=-1)
236
+ finally:
237
+ if swapped:
238
+ swap_out = _exchange(patched, corr_patcher, corrector=False)
239
+ log.info("H3 corrector(T) %s step %d: reloaded H3 in %.2fs", name, index, swap_out)
240
+
241
+ patched = model.clone()
242
+ if not swap_models:
243
+ patched.set_additional_models("h3_corrector", [corr_patcher])
244
+ patched.set_model_sampler_post_cfg_function(post_cfg)
245
+ return (patched,)
246
+
247
+ NODE_CLASS_MAPPINGS = {"LoadH3Corrector": LoadH3Corrector, "ApplyH3Corrector": ApplyH3Corrector, "ApplyH3CorrectorTunable": ApplyH3CorrectorTunable}
248
+ NODE_DISPLAY_NAME_MAPPINGS = {"LoadH3Corrector": "Load H3 Corrector", "ApplyH3Corrector": "Apply H3 Corrector", "ApplyH3CorrectorTunable": "Apply H3 Corrector (Tunable)"}
comfy/ComfyUI-H3-Corrector/corrector.py ADDED
@@ -0,0 +1,196 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """The H3 unified corrector network, inference only.
2
+
3
+ A 55-block space-time transformer over 2x2 video-latent patches. Each block runs joint self-attention over all video
4
+ tokens (with T/Y/X rotary positions), cross-attention to the caption states, and an MLP, modulated by an embedding of
5
+ the step's sigma interval. It reads the noisy latent and H3's velocity and predicts a velocity residual.
6
+ """
7
+ from __future__ import annotations
8
+
9
+ import dataclasses
10
+ import json
11
+ import math
12
+
13
+ import torch
14
+ import torch.nn.functional as F
15
+ from torch import nn
16
+
17
+ METADATA_KEY = "h3_corrector"
18
+
19
+
20
+ @dataclasses.dataclass(frozen=True)
21
+ class CorrectorConfig:
22
+ video_channels: int = 24
23
+ text_dim: int = 5120
24
+ width: int = 1280
25
+ depth: int = 55
26
+ heads: int = 20
27
+ head_dim: int = 64
28
+ mlp_ratio: int = 4
29
+ time_embed_dim: int = 256
30
+ patch: tuple[int, int, int] = (1, 2, 2)
31
+ norm_eps: float = 1e-5
32
+
33
+ def __post_init__(self):
34
+ if self.width != self.heads * self.head_dim:
35
+ raise ValueError("width must equal heads * head_dim")
36
+
37
+ @classmethod
38
+ def from_metadata(cls, metadata):
39
+ if not metadata or METADATA_KEY not in metadata:
40
+ raise ValueError(f"not an H3 corrector: no {METADATA_KEY!r} entry in the safetensors metadata")
41
+ raw = json.loads(metadata[METADATA_KEY])
42
+ return cls(**{**raw, "patch": tuple(raw["patch"])})
43
+
44
+ def to_metadata(self):
45
+ return {METADATA_KEY: json.dumps(dataclasses.asdict(self), separators=(",", ":"))}
46
+
47
+
48
+ def _time_embedding(sigma, width, max_period=10_000.0):
49
+ half = width // 2
50
+ frequencies = torch.exp(-math.log(max_period) * torch.arange(half, device=sigma.device, dtype=torch.float32) / half)
51
+ angles = sigma.float()[:, None] * frequencies[None] * 1000.0
52
+ embedding = torch.cat((torch.cos(angles), torch.sin(angles)), dim=-1)
53
+ return F.pad(embedding, (0, width - embedding.shape[-1]))
54
+
55
+
56
+ def _apply_rope_segment(x, positions, start, width):
57
+ """Rotate channels start..start+width of x [B,H,L,D] by positions [L]; consecutive pairs are complex coordinates."""
58
+ if width == 0:
59
+ return x
60
+ segment = x[..., start:start + width]
61
+ pairs = segment.reshape(*segment.shape[:-1], width // 2, 2)
62
+ inv = torch.exp(-math.log(10_000.0) * torch.arange(width // 2, device=x.device, dtype=torch.float32) / max(1, width // 2))
63
+ angle = positions.to(device=x.device, dtype=torch.float32)[:, None] * inv[None]
64
+ cos = angle.cos().to(dtype=x.dtype)[None, None]
65
+ sin = angle.sin().to(dtype=x.dtype)[None, None]
66
+ first, second = pairs[..., 0], pairs[..., 1]
67
+ rotated = torch.stack((first * cos - second * sin, first * sin + second * cos), dim=-1).flatten(-2)
68
+ return torch.cat((x[..., :start], rotated, x[..., start + width:]), dim=-1)
69
+
70
+
71
+ class JointSelfAttention(nn.Module):
72
+ """Attention over every video token, with independent T/Y/X rotary positions."""
73
+
74
+ def __init__(self, config):
75
+ super().__init__()
76
+ self.heads, self.head_dim = config.heads, config.head_dim
77
+ self.qkv = nn.Linear(config.width, 3 * config.width)
78
+ self.out = nn.Linear(config.width, config.width)
79
+
80
+ def forward(self, hidden):
81
+ batch, frames, height, width, dim = hidden.shape
82
+ rows = hidden.reshape(batch, frames * height * width, dim)
83
+ qkv = self.qkv(rows).reshape(batch, rows.shape[1], 3, self.heads, self.head_dim)
84
+ q, k, v = [part.transpose(1, 2) for part in qkv.unbind(2)]
85
+ t = torch.arange(frames, device=hidden.device).repeat_interleave(height * width)
86
+ y = torch.arange(height, device=hidden.device).repeat_interleave(width).repeat(frames)
87
+ x = torch.arange(width, device=hidden.device).repeat(frames * height)
88
+ temporal_width = 2 * (self.head_dim // 8)
89
+ height_width = 2 * ((self.head_dim - temporal_width) // 4)
90
+ widths = (temporal_width, height_width, self.head_dim - temporal_width - height_width)
91
+ offset = 0
92
+ for positions, segment_width in zip((t, y, x), widths):
93
+ q = _apply_rope_segment(q, positions, offset, segment_width)
94
+ k = _apply_rope_segment(k, positions, offset, segment_width)
95
+ offset += segment_width
96
+ values = F.scaled_dot_product_attention(q, k, v, dropout_p=0.0)
97
+ return self.out(values.transpose(1, 2).reshape(batch, frames, height, width, dim))
98
+
99
+
100
+ class CrossAttention(nn.Module):
101
+ def __init__(self, config):
102
+ super().__init__()
103
+ self.heads, self.head_dim = config.heads, config.head_dim
104
+ self.q = nn.Linear(config.width, config.width)
105
+ self.k = nn.Linear(config.width, config.width)
106
+ self.v = nn.Linear(config.width, config.width)
107
+ self.out = nn.Linear(config.width, config.width)
108
+
109
+ def forward(self, hidden, text):
110
+ batch, frames, height, width, dim = hidden.shape
111
+ rows = hidden.reshape(batch, frames * height * width, dim)
112
+ q = self.q(rows).reshape(batch, rows.shape[1], self.heads, self.head_dim).transpose(1, 2)
113
+ k = self.k(text).reshape(batch, text.shape[1], self.heads, self.head_dim).transpose(1, 2)
114
+ v = self.v(text).reshape(batch, text.shape[1], self.heads, self.head_dim).transpose(1, 2)
115
+ attended = F.scaled_dot_product_attention(q, k, v, dropout_p=0.0)
116
+ return self.out(attended.transpose(1, 2).reshape(batch, rows.shape[1], dim)).reshape(batch, frames, height, width, dim)
117
+
118
+
119
+ class FeedForward(nn.Module):
120
+ def __init__(self, config):
121
+ super().__init__()
122
+ self.fc1 = nn.Linear(config.width, config.width * config.mlp_ratio)
123
+ self.fc2 = nn.Linear(config.width * config.mlp_ratio, config.width)
124
+
125
+ def forward(self, hidden):
126
+ return self.fc2(F.gelu(self.fc1(hidden), approximate="tanh"))
127
+
128
+
129
+ class Block(nn.Module):
130
+ def __init__(self, config):
131
+ super().__init__()
132
+ self.norm_self = nn.LayerNorm(config.width, eps=config.norm_eps)
133
+ self.norm_cross = nn.LayerNorm(config.width, eps=config.norm_eps)
134
+ self.norm_mlp = nn.LayerNorm(config.width, eps=config.norm_eps)
135
+ self.self_attention = JointSelfAttention(config)
136
+ self.cross_attention = CrossAttention(config)
137
+ self.mlp = FeedForward(config)
138
+ self.modulation = nn.Linear(config.width, 6 * config.width)
139
+
140
+ def forward(self, hidden, time, text):
141
+ shift_sa, scale_sa, gate_sa, shift_ff, scale_ff, gate_ff = self.modulation(time).chunk(6, dim=-1)
142
+ broadcast = (slice(None), None, None, None, slice(None))
143
+ self_input = self.norm_self(hidden) * (1.0 + scale_sa[broadcast]) + shift_sa[broadcast]
144
+ hidden = hidden + (1.0 + gate_sa[broadcast]) * self.self_attention(self_input)
145
+ hidden = hidden + self.cross_attention(self.norm_cross(hidden), text)
146
+ mlp_input = self.norm_mlp(hidden) * (1.0 + scale_ff[broadcast]) + shift_ff[broadcast]
147
+ return hidden + (1.0 + gate_ff[broadcast]) * self.mlp(mlp_input)
148
+
149
+
150
+ class H3Corrector(nn.Module):
151
+ def __init__(self, config: CorrectorConfig):
152
+ super().__init__()
153
+ self.config = config
154
+ self.patch_in = nn.Conv3d(2 * config.video_channels, config.width, kernel_size=config.patch, stride=config.patch)
155
+ self.text_norm = nn.LayerNorm(config.text_dim, eps=config.norm_eps)
156
+ self.text_projection = nn.Linear(config.text_dim, config.width)
157
+ self.time_mlp = nn.Sequential(nn.Linear(3 * config.time_embed_dim, config.width), nn.SiLU(), nn.Linear(config.width, config.width))
158
+ self.blocks = nn.ModuleList(Block(config) for _ in range(config.depth))
159
+ self.final_norm = nn.LayerNorm(config.width, eps=config.norm_eps)
160
+ self.final_modulation = nn.Linear(config.width, 2 * config.width)
161
+ self.patch_out = nn.Linear(config.width, config.video_channels * math.prod(config.patch))
162
+
163
+ def forward(self, video_latents, velocity, sigma, sigma_next, text):
164
+ """Velocity residual for the step sigma -> sigma_next. video_latents/velocity: [B,C,T,H,W]; text: [B,L,text_dim]."""
165
+ batch, channels, frames, height, width = video_latents.shape
166
+ pt, ph, pw = self.config.patch
167
+ if velocity.shape != video_latents.shape or channels != self.config.video_channels or frames % pt or height % ph or width % pw:
168
+ raise ValueError(f"latent {tuple(video_latents.shape)} does not fit the corrector")
169
+ before = torch.as_tensor(sigma, device=video_latents.device, dtype=torch.float32).reshape(-1).expand(batch)
170
+ after = torch.as_tensor(sigma_next, device=video_latents.device, dtype=torch.float32).reshape(-1).expand(batch)
171
+ delta = before - after
172
+ hidden = self.patch_in(torch.cat((video_latents, velocity), dim=1)).permute(0, 2, 3, 4, 1)
173
+ text = self.text_projection(self.text_norm(text))
174
+ time = self.time_mlp(torch.cat([_time_embedding(s, self.config.time_embed_dim) for s in (before, after, delta)],
175
+ dim=-1).to(hidden.dtype))
176
+ for block in self.blocks:
177
+ hidden = block(hidden, time, text)
178
+ shift, scale = self.final_modulation(time).chunk(2, dim=-1)
179
+ hidden = self.final_norm(hidden) * (1 + scale[:, None, None, None]) + shift[:, None, None, None]
180
+ patches = self.patch_out(hidden).reshape(batch, frames // pt, height // ph, width // pw, channels, pt, ph, pw)
181
+ return patches.permute(0, 4, 1, 5, 2, 6, 3, 7).reshape(batch, channels, frames, height, width)
182
+
183
+
184
+ def bound_residual(raw, velocity, budget_fraction, velocity_floor, eps=1e-12):
185
+ """Soft-cap the residual's RMS at budget_fraction * max(RMS(velocity), velocity_floor), per video.
186
+
187
+ Returns (applied residual, applied RMS / velocity RMS).
188
+ """
189
+ raw, velocity = raw.float(), velocity.float()
190
+ dims = tuple(range(1, raw.ndim))
191
+ base_rms = velocity.square().mean(dim=dims).sqrt()
192
+ budget = budget_fraction * torch.clamp_min(base_rms, velocity_floor)
193
+ scale = torch.rsqrt(1.0 + raw.square().mean(dim=dims) / budget.square().clamp_min(eps))
194
+ applied = raw * scale.reshape(-1, *([1] * (raw.ndim - 1)))
195
+ ratio = applied.square().mean(dim=dims).sqrt() / base_rms.clamp_min(eps)
196
+ return applied, ratio