Spaces:
Running on Zero
Running on Zero
Upload folder using huggingface_hub
Browse files- README.md +46 -7
- app.py +365 -0
- lot/adapter.py +286 -0
- lot/flow.py +314 -0
- lot/gpu_guard.py +61 -0
- lot/h3.py +180 -0
- lot/h3_musubi.py +356 -0
- lot/h3_positions.py +168 -0
- lot/layout.py +385 -0
- lot/layout_sources.py +202 -0
- lot/lot_paths.py +36 -0
- lot/mixed_res.py +513 -0
- lot/procrustes_h3.py +223 -0
- lot/two_track_h3.py +946 -0
- requirements.txt +15 -0
- third_party/musubi_tuner/__init__.py +0 -0
- third_party/musubi_tuner/convert_lora.py +229 -0
- third_party/musubi_tuner/dataset/__init__.py +0 -0
- third_party/musubi_tuner/dataset/audio_utils.py +512 -0
- third_party/musubi_tuner/dataset/datasources.py +867 -0
- third_party/musubi_tuner/dataset/media_utils.py +415 -0
- third_party/musubi_tuner/minimax_h3/__init__.py +1 -0
- third_party/musubi_tuner/minimax_h3/checkpoint.py +65 -0
- third_party/musubi_tuner/minimax_h3/media.py +620 -0
- third_party/musubi_tuner/minimax_h3/model.py +1298 -0
- third_party/musubi_tuner/minimax_h3/packing.py +711 -0
- third_party/musubi_tuner/minimax_h3/text_encoder.py +712 -0
- third_party/musubi_tuner/minimax_h3/video_vae.py +727 -0
- third_party/musubi_tuner/modules/__init__.py +0 -0
- third_party/musubi_tuner/modules/attention.py +351 -0
- third_party/musubi_tuner/modules/comfy_quant_utils.py +62 -0
- third_party/musubi_tuner/modules/convrot_int8_kernels.py +546 -0
- third_party/musubi_tuner/modules/convrot_int8_utils.py +451 -0
- third_party/musubi_tuner/modules/custom_offloading_utils.py +1184 -0
- third_party/musubi_tuner/modules/fp8_optimization_utils.py +565 -0
- third_party/musubi_tuner/modules/nvfp4_utils.py +518 -0
- third_party/musubi_tuner/networks/__init__.py +0 -0
- third_party/musubi_tuner/networks/lora.py +1024 -0
- third_party/musubi_tuner/networks/lora_minimax_h3.py +99 -0
- third_party/musubi_tuner/utils/__init__.py +0 -0
- third_party/musubi_tuner/utils/cv2_compat.py +59 -0
- third_party/musubi_tuner/utils/device_utils.py +28 -0
- third_party/musubi_tuner/utils/lora_utils.py +368 -0
- third_party/musubi_tuner/utils/model_utils.py +342 -0
- third_party/musubi_tuner/utils/safetensors_utils.py +483 -0
README.md
CHANGED
|
@@ -1,13 +1,52 @@
|
|
| 1 |
---
|
| 2 |
-
title:
|
| 3 |
-
emoji:
|
| 4 |
-
colorFrom:
|
| 5 |
-
colorTo:
|
| 6 |
sdk: gradio
|
| 7 |
sdk_version: 6.32.0
|
| 8 |
-
python_version: '3.12'
|
| 9 |
app_file: app.py
|
| 10 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 11 |
---
|
| 12 |
|
| 13 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
+
title: MPEG-Diffusion
|
| 3 |
+
emoji: 🎞️
|
| 4 |
+
colorFrom: purple
|
| 5 |
+
colorTo: green
|
| 6 |
sdk: gradio
|
| 7 |
sdk_version: 6.32.0
|
|
|
|
| 8 |
app_file: app.py
|
| 9 |
+
python_version: "3.12"
|
| 10 |
+
startup_duration_timeout: 1h
|
| 11 |
+
short_description: Mixed-resolution text-to-video with a self-selected region
|
| 12 |
+
models:
|
| 13 |
+
- johndpope/MiniMax-H3-MPEG-Diffusion
|
| 14 |
---
|
| 15 |
|
| 16 |
+
# MPEG-Diffusion: Mixed-Resolution Video Generation
|
| 17 |
+
|
| 18 |
+
Text-to-video demo for **[MPEG-Diffusion](https://huggingface.co/papers/2611.12069)**
|
| 19 |
+
(Menn et al.) — a model that generates video the way video codecs think about it:
|
| 20 |
+
|
| 21 |
+
1. a **low-resolution track** denoises the whole frame cheaply,
|
| 22 |
+
2. a tiny linear **region head** self-selects the region that deserves detail,
|
| 23 |
+
3. a **high-resolution track** refines only that region through dual-resolution attention,
|
| 24 |
+
4. the two tracks **merge in pixel space** — full-res detail where it matters, at roughly
|
| 25 |
+
half the compute of rendering everything at high resolution.
|
| 26 |
+
|
| 27 |
+
This Space runs the paper's [MiniMax-H3 port](https://github.com/scrya-com/mpeg-diffusion)
|
| 28 |
+
(Apache-2.0, vendored under `lot/` + `third_party/`) with the published research weights from
|
| 29 |
+
[johndpope/MiniMax-H3-MPEG-Diffusion](https://huggingface.co/johndpope/MiniMax-H3-MPEG-Diffusion):
|
| 30 |
+
the ridge-fitted KV up-maps and the v3 region head applied to the int8 ConvRot base DiT
|
| 31 |
+
([Comfy-Org/MiniMax-H3](https://huggingface.co/Comfy-Org/MiniMax-H3)), at the port's
|
| 32 |
+
*quality preset* — 20 steps, the region chosen at step 16, 6 high-resolution steps, pixel-space
|
| 33 |
+
merge. By default the model picks its own detail region; a manual box can be forced under
|
| 34 |
+
*Advanced*.
|
| 35 |
+
|
| 36 |
+
Because the 62 GiB Qwen3-VL text encoder does not fit beside the transformer on one GPU,
|
| 37 |
+
prompt conditioning runs on the
|
| 38 |
+
[qwen3vl-conditioner](https://huggingface.co/spaces/multimodalart/qwen3vl-conditioner)
|
| 39 |
+
Space and this Space runs the sampler.
|
| 40 |
+
|
| 41 |
+
## Credits
|
| 42 |
+
|
| 43 |
+
- **Paper**: *MPEG-Diffusion: Mixed-Resolution Video Generation* — Menn et al., arXiv 2611.12069.
|
| 44 |
+
- **Code**: the paper's MiniMax-H3 port, [scrya-com/mpeg-diffusion](https://github.com/scrya-com/mpeg-diffusion) (Apache-2.0).
|
| 45 |
+
- **Weights**: [johndpope/MiniMax-H3-MPEG-Diffusion](https://huggingface.co/johndpope/MiniMax-H3-MPEG-Diffusion) — fitted up-maps + region head on the int8 ConvRot base.
|
| 46 |
+
- **Base model**: MiniMax-H3, subject to the
|
| 47 |
+
[MiniMax H3 Community License](https://huggingface.co/Comfy-Org/MiniMax-H3) — outputs are
|
| 48 |
+
machine-generated and must be disclosed as such; the license does not permit use in the
|
| 49 |
+
EU, UK, Republic of Korea or USA, and commercial use by companies above US$20M annual
|
| 50 |
+
revenue needs written authorization from MiniMax.
|
| 51 |
+
- **Example prompts**: portrait descriptions in the style of Pexels image alt-texts
|
| 52 |
+
(Pexels content is free to use).
|
app.py
ADDED
|
@@ -0,0 +1,365 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""MPEG-Diffusion — mixed-resolution text-to-video generation on MiniMax-H3.
|
| 2 |
+
|
| 3 |
+
This Space implements the MPEG-Diffusion inference recipe end to end: a low-resolution track
|
| 4 |
+
denoises the whole frame, a linear region head self-selects where detail matters, and a
|
| 5 |
+
high-resolution track refines only that region through dual-resolution attention with
|
| 6 |
+
ridge-fitted up-maps, before the two tracks are merged in pixel space.
|
| 7 |
+
|
| 8 |
+
The reference implementation is the paper's port for MiniMax-H3
|
| 9 |
+
(github.com/scrya-com/mpeg-diffusion, Apache-2.0), vendored here as ``lot/`` plus a
|
| 10 |
+
subset of its musubi-tuner (``third_party/musubi_tuner/``). The weights are the published
|
| 11 |
+
research artifacts (johndpope/MiniMax-H3-MPEG-Diffusion): the fitted KV up-maps and the
|
| 12 |
+
v3 region head, applied to the *untrained* ComfyUI int8 ConvRot base DiT — exactly the
|
| 13 |
+
"quality preset" the port's README documents (`--preset quality`: steps 20, switch 16,
|
| 14 |
+
hr_steps 6, switch_upscale="crop", final="pixel", feather 4.0, alpha 0.5, block 25).
|
| 15 |
+
|
| 16 |
+
The 62 GiB Qwen3-VL text encoder does not fit beside the transformer on a single ZeroGPU
|
| 17 |
+
worker, so conditioning is delegated to ``multimodalart/qwen3vl-conditioner`` over the
|
| 18 |
+
gradio API — the same split every MiniMax-H3 Space in this org uses. Its ``/encode``
|
| 19 |
+
returns the layer-50 hidden state ``prompt_embeds (1, L, 5120) bfloat16``, which is
|
| 20 |
+
precisely the ``text`` input ``two_track_sample`` consumes.
|
| 21 |
+
"""
|
| 22 |
+
|
| 23 |
+
from __future__ import annotations
|
| 24 |
+
|
| 25 |
+
import os
|
| 26 |
+
|
| 27 |
+
# expandable_segments avoids the fragmentation the 21 GB int8 streaming load would otherwise
|
| 28 |
+
# hit; must be set before torch is imported. Same for the muted triton autotune cache dir.
|
| 29 |
+
os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
|
| 30 |
+
# The int8 Triton kernels autotune; keep the cache on the ephemeral disk, not a read-only layer.
|
| 31 |
+
os.environ.setdefault("TRITON_CACHE_DIR", "/tmp/triton-cache")
|
| 32 |
+
|
| 33 |
+
import spaces # noqa: E402 — FIRST, before any CUDA-touching import
|
| 34 |
+
|
| 35 |
+
import sys
|
| 36 |
+
import tempfile
|
| 37 |
+
import time
|
| 38 |
+
import traceback
|
| 39 |
+
from functools import cache
|
| 40 |
+
from pathlib import Path
|
| 41 |
+
|
| 42 |
+
import gradio as gr
|
| 43 |
+
import torch
|
| 44 |
+
|
| 45 |
+
SPACE_ROOT = Path(__file__).resolve().parent
|
| 46 |
+
|
| 47 |
+
# The weights the Space downloads at runtime. The paths are exported *before* any ``lot``
|
| 48 |
+
# import because ``lot_paths`` reads them at import time (``procrustes_h3`` reads
|
| 49 |
+
# ``LOT_H3_CHECKPOINT`` the moment it is imported).
|
| 50 |
+
BASE_REPO = "Comfy-Org/MiniMax-H3"
|
| 51 |
+
DIT_FILE = "diffusion_models/minimax_h3_fl2va_pruned_int8_convrot.safetensors"
|
| 52 |
+
VAE_FILE = "vae/minimax_h3_video_vae_fp16.safetensors"
|
| 53 |
+
TT_REPO = "johndpope/MiniMax-H3-MPEG-Diffusion"
|
| 54 |
+
MAPS_FILE = "kv_maps/up_maps_tt.safetensors"
|
| 55 |
+
REGION_FILE = "region_head_v3/region_head_tt.safetensors"
|
| 56 |
+
|
| 57 |
+
CONDITIONER_SPACE = os.environ.get("H3_CONDITIONER", "multimodalart/qwen3vl-conditioner")
|
| 58 |
+
|
| 59 |
+
# The quality preset from the port's README, pinned as defaults. 1344x768 is the canvas the
|
| 60 |
+
# maps and the head were fitted/validated on. Clip length must sit on the FRAME_PER_TOKEN
|
| 61 |
+
# grid (17n+5 pixel frames -> latent_t 5n+2) AND clear the conditioner's 2 s floor: 22 frames
|
| 62 |
+
# is 0.92 s and would be rejected by /encode, so the shortest clip that works is 56 frames
|
| 63 |
+
# (2.33 s at 24 fps -> latent_t 17).
|
| 64 |
+
CANVAS_W, CANVAS_H = 1344, 768
|
| 65 |
+
NUM_FRAMES = 56
|
| 66 |
+
FPS = 24
|
| 67 |
+
DEFAULT_STEPS, DEFAULT_SWITCH, DEFAULT_HR_STEPS = 20, 16, 6
|
| 68 |
+
DEFAULT_FEATHER, DEFAULT_ALPHA = 4.0, 0.5
|
| 69 |
+
|
| 70 |
+
OUTPUT_DIR = tempfile.mkdtemp(prefix="mpeg-diffusion-out-")
|
| 71 |
+
|
| 72 |
+
GPU_SIZE = "large"
|
| 73 |
+
|
| 74 |
+
DIT: torch.nn.Module | None = None
|
| 75 |
+
UPSCALE = None # lot.two_track_h3.VaeUpscaler
|
| 76 |
+
MAPS = None # lot.mixed_res.UpMaps
|
| 77 |
+
HEAD = None # lot.mixed_res.RegionHead
|
| 78 |
+
HEAD_BLOCK = 25
|
| 79 |
+
LOAD_ERROR: str | None = None
|
| 80 |
+
LOADED_IN: float | None = None
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
# ── Model loading (module scope, eager) ─────────────────────────────────────
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
def load_models() -> str | None:
|
| 87 |
+
"""Load DiT + VAE + up-maps + region head at startup, `.to("cuda")` so the ZeroGPU
|
| 88 |
+
startup pack streams them disk->VRAM on a cold worker (26 GB total)."""
|
| 89 |
+
global DIT, UPSCALE, MAPS, HEAD, HEAD_BLOCK, LOAD_ERROR, LOADED_IN
|
| 90 |
+
if DIT is not None or LOAD_ERROR is not None:
|
| 91 |
+
return LOAD_ERROR
|
| 92 |
+
started = time.time()
|
| 93 |
+
try:
|
| 94 |
+
from huggingface_hub import hf_hub_download
|
| 95 |
+
|
| 96 |
+
os.environ["LOT_H3_CHECKPOINT"] = hf_hub_download(BASE_REPO, DIT_FILE)
|
| 97 |
+
os.environ["LOT_H3_VAE"] = hf_hub_download(BASE_REPO, VAE_FILE)
|
| 98 |
+
maps_path = hf_hub_download(TT_REPO, MAPS_FILE)
|
| 99 |
+
region_path = hf_hub_download(TT_REPO, REGION_FILE)
|
| 100 |
+
|
| 101 |
+
sys.path.insert(0, str(SPACE_ROOT / "lot"))
|
| 102 |
+
sys.path.insert(0, str(SPACE_ROOT / "third_party"))
|
| 103 |
+
from two_track_h3 import load_maps, load_region, load_upscaler # lot/ is on sys.path
|
| 104 |
+
|
| 105 |
+
from h3_musubi import load_dit
|
| 106 |
+
|
| 107 |
+
print(f"[load] int8 ConvRot DiT from {BASE_REPO}/{DIT_FILE} ...", flush=True)
|
| 108 |
+
DIT = load_dit(os.environ["LOT_H3_CHECKPOINT"], device="cuda", swap=0)
|
| 109 |
+
print(f"[load] DiT resident ({time.time() - started:.0f}s), loading VAE ...", flush=True)
|
| 110 |
+
UPSCALE = load_upscaler()
|
| 111 |
+
print(f"[load] VAE resident, loading fitted up-maps + region head ...", flush=True)
|
| 112 |
+
MAPS = load_maps(Path(maps_path))
|
| 113 |
+
saved = load_region(Path(region_path))
|
| 114 |
+
from mixed_res import RegionHead
|
| 115 |
+
|
| 116 |
+
HEAD = RegionHead(saved["hidden"]).cuda().eval()
|
| 117 |
+
HEAD.load_state_dict(saved["state"])
|
| 118 |
+
HEAD_BLOCK = int(saved["block"])
|
| 119 |
+
LOADED_IN = time.time() - started
|
| 120 |
+
print(f"[load] ready in {LOADED_IN:.0f}s (region head on block {HEAD_BLOCK})", flush=True)
|
| 121 |
+
except Exception as error:
|
| 122 |
+
traceback.print_exc()
|
| 123 |
+
LOAD_ERROR = f"**Loading failed** after {time.time() - started:.0f}s: `{type(error).__name__}: {error}`"
|
| 124 |
+
return LOAD_ERROR
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
load_models()
|
| 128 |
+
|
| 129 |
+
|
| 130 |
+
# ── Remote conditioning ──────────────────────────────────────────────────────
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
@cache
|
| 134 |
+
def conditioner():
|
| 135 |
+
from gradio_client import Client
|
| 136 |
+
|
| 137 |
+
return Client(CONDITIONER_SPACE)
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
def conditioner_client(ip_token):
|
| 141 |
+
if not ip_token:
|
| 142 |
+
return conditioner()
|
| 143 |
+
from gradio_client import Client
|
| 144 |
+
|
| 145 |
+
return Client(CONDITIONER_SPACE, headers={"x-ip-token": ip_token})
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
def encode_remote(prompt, ip_token=None):
|
| 149 |
+
"""Call the conditioner Space for the layer-50 Qwen3-VL states ``two_track_sample`` consumes."""
|
| 150 |
+
from safetensors import safe_open
|
| 151 |
+
|
| 152 |
+
def call():
|
| 153 |
+
return conditioner_client(ip_token).predict(
|
| 154 |
+
prompt=prompt,
|
| 155 |
+
image_path=None,
|
| 156 |
+
last_image_path=None,
|
| 157 |
+
canvas=f"{CANVAS_W}x{CANVAS_H} · 16:9 full",
|
| 158 |
+
num_frames=NUM_FRAMES,
|
| 159 |
+
rewrite_prompt=False,
|
| 160 |
+
api_name="/encode",
|
| 161 |
+
)
|
| 162 |
+
|
| 163 |
+
try:
|
| 164 |
+
path, plan = call()
|
| 165 |
+
except Exception as first:
|
| 166 |
+
print(f"[cond] retrying with a fresh client after: {first}", flush=True)
|
| 167 |
+
conditioner.cache_clear()
|
| 168 |
+
path, plan = call()
|
| 169 |
+
|
| 170 |
+
with safe_open(path, framework="pt") as handle:
|
| 171 |
+
return handle.get_tensor("prompt_embeds"), plan
|
| 172 |
+
|
| 173 |
+
|
| 174 |
+
def _caller_ip_token() -> str | None:
|
| 175 |
+
from gradio.context import LocalContext
|
| 176 |
+
|
| 177 |
+
request = LocalContext.request.get()
|
| 178 |
+
return request.headers.get("x-ip-token") if request is not None else None
|
| 179 |
+
|
| 180 |
+
|
| 181 |
+
# ── GPU inference ─────────────────────────────────────────────────────────────
|
| 182 |
+
|
| 183 |
+
|
| 184 |
+
@spaces.GPU(duration=240, size=GPU_SIZE)
|
| 185 |
+
def _generate(prompt_embeds, seed, steps, switch, hr_steps, feather, alpha, use_head, box):
|
| 186 |
+
"""Run the four-stage two-track sampler and compose to pixels, all on GPU time.
|
| 187 |
+
|
| 188 |
+
Mirrors ``two_track_h3.render()`` with ``--preset quality``: 20 steps of which the
|
| 189 |
+
first 16 run at half resolution, the region head picks M from block-25 states at the
|
| 190 |
+
switch, the HR track takes 6 linear-sigma steps on M only (``switch_upscale="crop"``
|
| 191 |
+
round-trips just the bounding box through the VAE), and the merge happens in pixel
|
| 192 |
+
space (``final="pixel"``) with a feather of 4 cells.
|
| 193 |
+
|
| 194 |
+
``two_track_sample`` has three return shapes: when the region comes back empty (or
|
| 195 |
+
switch >= steps) it returns only the plain LR latent, so the pixel merge must be
|
| 196 |
+
guarded exactly the way ``render()`` guards it.
|
| 197 |
+
"""
|
| 198 |
+
import torch
|
| 199 |
+
|
| 200 |
+
from two_track_h3 import pixel_compose, save_clip, two_track_sample
|
| 201 |
+
from h3_musubi import latent_frames
|
| 202 |
+
|
| 203 |
+
latent_t = latent_frames(NUM_FRAMES)
|
| 204 |
+
head = HEAD if use_head else None
|
| 205 |
+
box = tuple(float(v) for v in box) if (box and not use_head) else None
|
| 206 |
+
|
| 207 |
+
embeds = prompt_embeds.to("cuda", torch.bfloat16)
|
| 208 |
+
|
| 209 |
+
latent, info = two_track_sample(
|
| 210 |
+
DIT, embeds, UPSCALE,
|
| 211 |
+
height=CANVAS_H, width=CANVAS_W, steps=int(steps), switch=int(switch), seed=int(seed),
|
| 212 |
+
maps=MAPS, head=head, box=box, block=HEAD_BLOCK,
|
| 213 |
+
alpha=float(alpha), feather=float(feather), hr_steps=int(hr_steps),
|
| 214 |
+
latent_t=latent_t, switch_upscale="crop", final="pixel",
|
| 215 |
+
)
|
| 216 |
+
if "lr" in info: # render()'s guard: the merge inputs exist only on the two-track path
|
| 217 |
+
pixels = pixel_compose(UPSCALE.vae, latent, info["lr"], info["weight"])
|
| 218 |
+
else:
|
| 219 |
+
pixels = UPSCALE.vae.decode(latent.to("cuda", UPSCALE.dtype)).to("cpu", torch.float16)
|
| 220 |
+
out_path = os.path.join(OUTPUT_DIR, f"mpeg-diffusion-{int(time.time() * 1000)}.mp4")
|
| 221 |
+
save_clip(pixels, Path(out_path), fps=FPS)
|
| 222 |
+
share = float(info["mask"].float().mean()) if info.get("mask") is not None else 0.0
|
| 223 |
+
return out_path, share
|
| 224 |
+
|
| 225 |
+
|
| 226 |
+
def generate(
|
| 227 |
+
prompt,
|
| 228 |
+
use_head=True,
|
| 229 |
+
box_x0=0.25,
|
| 230 |
+
box_y0=0.25,
|
| 231 |
+
box_x1=0.75,
|
| 232 |
+
box_y1=0.75,
|
| 233 |
+
seed=0,
|
| 234 |
+
steps=DEFAULT_STEPS,
|
| 235 |
+
switch=DEFAULT_SWITCH,
|
| 236 |
+
hr_steps=DEFAULT_HR_STEPS,
|
| 237 |
+
feather=DEFAULT_FEATHER,
|
| 238 |
+
alpha=DEFAULT_ALPHA,
|
| 239 |
+
progress=gr.Progress(track_tqdm=True),
|
| 240 |
+
):
|
| 241 |
+
"""One request: prompt -> mixed-resolution video. Conditioning runs on the conditioner
|
| 242 |
+
Space (outside GPU booking, charged to the caller's quota via x-ip-token); the sampler
|
| 243 |
+
runs on this Space's ZeroGPU worker."""
|
| 244 |
+
if LOAD_ERROR:
|
| 245 |
+
raise gr.Error(LOAD_ERROR.replace("**", "").replace("`", ""))
|
| 246 |
+
if DIT is None:
|
| 247 |
+
raise gr.Error("The model is still loading — watch the Space logs and retry shortly.")
|
| 248 |
+
if not prompt or not prompt.strip():
|
| 249 |
+
raise gr.Error("A prompt is required.")
|
| 250 |
+
if int(switch) >= int(steps):
|
| 251 |
+
raise gr.Error("The region switch step must come before the last step.")
|
| 252 |
+
box = (float(box_x0), float(box_y0), float(box_x1), float(box_y1))
|
| 253 |
+
if box[0] >= box[2] or box[1] >= box[3]:
|
| 254 |
+
raise gr.Error("The manual region box needs x1 > x0 and y1 > y0.")
|
| 255 |
+
|
| 256 |
+
progress(0.0, desc="Encoding the prompt ...")
|
| 257 |
+
prompt_embeds, plan = encode_remote(prompt, ip_token=_caller_ip_token())
|
| 258 |
+
|
| 259 |
+
progress(0.1, desc=f"Two-track sampling {CANVAS_W}x{CANVAS_H} x {NUM_FRAMES} frames ...")
|
| 260 |
+
started = time.time()
|
| 261 |
+
out_path, share = _generate(
|
| 262 |
+
prompt_embeds, int(seed), int(steps), int(switch), int(hr_steps),
|
| 263 |
+
float(feather), float(alpha), bool(use_head), box,
|
| 264 |
+
)
|
| 265 |
+
seconds = time.time() - started
|
| 266 |
+
|
| 267 |
+
region = "region head" if use_head else f"manual box {tuple(round(v, 2) for v in box)}"
|
| 268 |
+
report = (
|
| 269 |
+
f"{CANVAS_W}x{CANVAS_H} · {NUM_FRAMES} frames ({NUM_FRAMES / FPS:.2f}s) · "
|
| 270 |
+
f"{int(steps)} steps, switch at {int(switch)}, {int(hr_steps)} HR steps · "
|
| 271 |
+
f"HR region: {region} covering {share:.0%} of tokens · "
|
| 272 |
+
f"conditioner {plan.get('seconds', '?')}s · sampler {seconds:.0f}s · seed {int(seed)}"
|
| 273 |
+
)
|
| 274 |
+
print(f"[gen] {report}", flush=True)
|
| 275 |
+
return out_path, report
|
| 276 |
+
|
| 277 |
+
|
| 278 |
+
# ── UI ────────────────────────────────────────────────────────────────────────
|
| 279 |
+
|
| 280 |
+
|
| 281 |
+
INTRO = """# MPEG-Diffusion: Mixed-Resolution Video Generation
|
| 282 |
+
|
| 283 |
+
<div align="center">
|
| 284 |
+
<a href="https://huggingface.co/papers/2611.12069" target="_blank" rel="noopener"><strong>[ paper ]</strong></a>
|
| 285 |
+
<a href="https://github.com/scrya-com/mpeg-diffusion" target="_blank" rel="noopener"><strong>[ code ]</strong></a>
|
| 286 |
+
<a href="https://huggingface.co/johndpope/MiniMax-H3-MPEG-Diffusion" target="_blank" rel="noopener"><strong>[ weights ]</strong></a>
|
| 287 |
+
</div>
|
| 288 |
+
|
| 289 |
+
**MPEG-Diffusion** (Menn et al.) generates video the way video codecs think about it: a
|
| 290 |
+
**low-resolution track** denoises the whole frame cheaply, a tiny linear head **self-selects
|
| 291 |
+
the region that deserves detail**, and a **high-resolution track** refines only that region
|
| 292 |
+
through dual-resolution attention. The tracks merge in pixel space — full-res detail where it
|
| 293 |
+
matters, ~2x the speed of rendering everything at high resolution.
|
| 294 |
+
|
| 295 |
+
This Space runs the paper's MiniMax-H3 port with the published research weights: the fitted
|
| 296 |
+
KV up-maps and the v3 region head on the int8 ConvRot base DiT, at the port's *quality preset*
|
| 297 |
+
(20 steps, region chosen at step 16, 6 HR steps, pixel-space merge). By default the model
|
| 298 |
+
picks its own detail region; force one with a manual box in *Advanced*."""
|
| 299 |
+
|
| 300 |
+
CSS = """
|
| 301 |
+
.main.fillable { max-width: 1100px !important; }
|
| 302 |
+
"""
|
| 303 |
+
|
| 304 |
+
EXAMPLE_PROMPTS = [
|
| 305 |
+
"A portrait of a young woman holding a red hibiscus flower against a rustic background",
|
| 306 |
+
"A joyful woman with colorful flowers in her afro, exuding happiness and vibrant style",
|
| 307 |
+
"A bearded man in a bathrobe uses a smartphone under studio lighting with a soft focus",
|
| 308 |
+
"A fashionable man in checkered pants and cowboy hat poses outdoors on concrete blocks",
|
| 309 |
+
"A serene portrait of a woman with curly hair surrounded by autumn foliage and berries",
|
| 310 |
+
]
|
| 311 |
+
|
| 312 |
+
with gr.Blocks(title="MPEG-Diffusion") as demo:
|
| 313 |
+
gr.Markdown(INTRO)
|
| 314 |
+
|
| 315 |
+
with gr.Row(equal_height=True):
|
| 316 |
+
with gr.Column():
|
| 317 |
+
prompt = gr.Textbox(
|
| 318 |
+
label="Prompt",
|
| 319 |
+
lines=3,
|
| 320 |
+
value=EXAMPLE_PROMPTS[0],
|
| 321 |
+
placeholder="Describe the scene — a clear subject helps the region head pick well ...",
|
| 322 |
+
)
|
| 323 |
+
use_head = gr.Checkbox(
|
| 324 |
+
label="Let the model choose the detail region (recommended)",
|
| 325 |
+
value=True,
|
| 326 |
+
)
|
| 327 |
+
run = gr.Button("Generate", variant="primary", size="lg")
|
| 328 |
+
|
| 329 |
+
with gr.Accordion("Advanced", open=False):
|
| 330 |
+
seed = gr.Number(label="Seed", value=0, precision=0)
|
| 331 |
+
with gr.Row():
|
| 332 |
+
steps = gr.Slider(label="Steps", minimum=10, maximum=30, step=1, value=DEFAULT_STEPS)
|
| 333 |
+
switch = gr.Slider(label="Region switch step", minimum=8, maximum=24, step=1, value=DEFAULT_SWITCH)
|
| 334 |
+
with gr.Row():
|
| 335 |
+
hr_steps = gr.Slider(label="HR steps", minimum=2, maximum=10, step=1, value=DEFAULT_HR_STEPS)
|
| 336 |
+
feather = gr.Slider(label="Feather (cells)", minimum=0, maximum=8, step=1, value=DEFAULT_FEATHER)
|
| 337 |
+
with gr.Row():
|
| 338 |
+
alpha = gr.Slider(label="Region threshold α", minimum=0.1, maximum=0.9, step=0.05, value=DEFAULT_ALPHA)
|
| 339 |
+
with gr.Accordion("Manual region (used when the checkbox is off)", open=False):
|
| 340 |
+
with gr.Row():
|
| 341 |
+
box_x0 = gr.Slider(label="Left x0", minimum=0.0, maximum=1.0, step=0.01, value=0.25)
|
| 342 |
+
box_y0 = gr.Slider(label="Top y0", minimum=0.0, maximum=1.0, step=0.01, value=0.25)
|
| 343 |
+
with gr.Row():
|
| 344 |
+
box_x1 = gr.Slider(label="Right x1", minimum=0.0, maximum=1.0, step=0.01, value=0.75)
|
| 345 |
+
box_y1 = gr.Slider(label="Bottom y1", minimum=0.0, maximum=1.0, step=0.01, value=0.75)
|
| 346 |
+
|
| 347 |
+
with gr.Column():
|
| 348 |
+
result = gr.Video(label="Mixed-resolution video", autoplay=True)
|
| 349 |
+
report = gr.Markdown()
|
| 350 |
+
|
| 351 |
+
gr.Examples(
|
| 352 |
+
examples=[[p] for p in EXAMPLE_PROMPTS],
|
| 353 |
+
inputs=[prompt],
|
| 354 |
+
fn=generate,
|
| 355 |
+
outputs=[result, report],
|
| 356 |
+
cache_examples=True,
|
| 357 |
+
cache_mode="lazy",
|
| 358 |
+
)
|
| 359 |
+
|
| 360 |
+
inputs = [prompt, use_head, box_x0, box_y0, box_x1, box_y1, seed, steps, switch, hr_steps, feather, alpha]
|
| 361 |
+
run.click(generate, inputs, [result, report], api_name="generate")
|
| 362 |
+
prompt.submit(generate, inputs, [result, report], api_name="generate_submit")
|
| 363 |
+
|
| 364 |
+
if __name__ == "__main__":
|
| 365 |
+
demo.launch(show_error=True, theme=gr.themes.Citrus(), css=CSS, allowed_paths=[OUTPUT_DIR])
|
lot/adapter.py
ADDED
|
@@ -0,0 +1,286 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Level-of-Token DiT adapter.
|
| 2 |
+
|
| 3 |
+
The backbone still runs attention. This module replaces only the visual
|
| 4 |
+
token pack, the positional centers, and the velocity head:
|
| 5 |
+
|
| 6 |
+
tokens (B, T, H, W, D) in scaled y-space
|
| 7 |
+
-> per-extent input head + shape MLP (eqs. 11–12)
|
| 8 |
+
-> backbone(hidden, centers, sigma) -> hidden
|
| 9 |
+
-> per-extent output head (eq. 10)
|
| 10 |
+
-> LakonLab asymmetric-velocity recovery (eq. 9)
|
| 11 |
+
-> dense velocity on the uniform lattice
|
| 12 |
+
|
| 13 |
+
Text, audio, and the transformer's own blocks stay inside ``backbone``.
|
| 14 |
+
It must return one hidden vector per LoT token, in ``layout.rects`` order.
|
| 15 |
+
"""
|
| 16 |
+
|
| 17 |
+
from __future__ import annotations
|
| 18 |
+
|
| 19 |
+
import torch
|
| 20 |
+
from torch import nn
|
| 21 |
+
|
| 22 |
+
from flow import (
|
| 23 |
+
apply_extent_scales,
|
| 24 |
+
asymmetric_target,
|
| 25 |
+
fit_procrustes,
|
| 26 |
+
gather_extent,
|
| 27 |
+
mean_basis,
|
| 28 |
+
recover_dense_velocity,
|
| 29 |
+
scatter_extent,
|
| 30 |
+
)
|
| 31 |
+
from layout import LotLayout, shape_features
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def _key(extent: tuple[int, int, int]) -> str:
|
| 35 |
+
et, eh, ew = extent
|
| 36 |
+
return f"{et}x{eh}x{ew}"
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
class ExtentBank(nn.Module):
|
| 40 |
+
"""Fixed per-extent lifts ``A`` and data scales ``s``.
|
| 41 |
+
|
| 42 |
+
Bases start as the mean lift and stay frozen. ``fit_`` replaces one
|
| 43 |
+
extent with a Procrustes fit; training does not update these buffers.
|
| 44 |
+
"""
|
| 45 |
+
|
| 46 |
+
def __init__(self, token_dim: int, extents: list[tuple[int, int, int]]):
|
| 47 |
+
super().__init__()
|
| 48 |
+
if not extents:
|
| 49 |
+
raise ValueError("at least one extent is required")
|
| 50 |
+
self.token_dim = int(token_dim)
|
| 51 |
+
self.extents = [tuple(map(int, extent)) for extent in extents]
|
| 52 |
+
seen = set()
|
| 53 |
+
for extent in self.extents:
|
| 54 |
+
if extent in seen:
|
| 55 |
+
raise ValueError(f"duplicate extent {extent}")
|
| 56 |
+
seen.add(extent)
|
| 57 |
+
if min(extent) < 1:
|
| 58 |
+
raise ValueError(f"bad extent {extent}")
|
| 59 |
+
key = _key(extent)
|
| 60 |
+
self.register_buffer(f"A_{key}", mean_basis(self.token_dim, extent))
|
| 61 |
+
self.register_buffer(f"s_{key}", torch.ones(()))
|
| 62 |
+
|
| 63 |
+
def basis(self, extent: tuple[int, int, int]) -> torch.Tensor:
|
| 64 |
+
return getattr(self, f"A_{_key(extent)}")
|
| 65 |
+
|
| 66 |
+
def scale(self, extent: tuple[int, int, int]) -> torch.Tensor:
|
| 67 |
+
return getattr(self, f"s_{_key(extent)}")
|
| 68 |
+
|
| 69 |
+
def scales(self) -> dict[tuple[int, int, int], torch.Tensor]:
|
| 70 |
+
return {extent: self.scale(extent) for extent in self.extents}
|
| 71 |
+
|
| 72 |
+
def dense_dim(self, extent: tuple[int, int, int]) -> int:
|
| 73 |
+
et, eh, ew = extent
|
| 74 |
+
return self.token_dim * et * eh * ew
|
| 75 |
+
|
| 76 |
+
@torch.no_grad()
|
| 77 |
+
def fit_(self, extent: tuple[int, int, int], dense: torch.Tensor, reference: torch.Tensor) -> None:
|
| 78 |
+
"""Replace ``A`` and ``s`` for one extent. Inputs are detached."""
|
| 79 |
+
if tuple(extent) not in self.extents:
|
| 80 |
+
raise KeyError(f"extent {extent} is not in this bank")
|
| 81 |
+
if dense.shape[-1] != self.dense_dim(extent) or reference.shape[-1] != self.token_dim:
|
| 82 |
+
raise ValueError(
|
| 83 |
+
f"expected dense (N, {self.dense_dim(extent)}) and reference (N, {self.token_dim})"
|
| 84 |
+
)
|
| 85 |
+
basis, scale = fit_procrustes(dense, reference)
|
| 86 |
+
slot = self.basis(extent)
|
| 87 |
+
slot.copy_(basis.to(device=slot.device, dtype=slot.dtype))
|
| 88 |
+
scale_slot = self.scale(extent)
|
| 89 |
+
scale_slot.copy_(scale.to(device=scale_slot.device, dtype=scale_slot.dtype))
|
| 90 |
+
|
| 91 |
+
def scale_clean(self, x0: torch.Tensor, layout: LotLayout) -> torch.Tensor:
|
| 92 |
+
"""Eq. 16. ``x0`` is ``(B, T, H, W, D)`` in the pretrained latent space."""
|
| 93 |
+
self._check_layout(layout)
|
| 94 |
+
return apply_extent_scales(x0, layout, self.scales(), invert=False)
|
| 95 |
+
|
| 96 |
+
def unscale(self, y0: torch.Tensor, layout: LotLayout) -> torch.Tensor:
|
| 97 |
+
"""Map a y-space clean estimate back to the pretrained latent."""
|
| 98 |
+
self._check_layout(layout)
|
| 99 |
+
return apply_extent_scales(y0, layout, self.scales(), invert=True)
|
| 100 |
+
|
| 101 |
+
def _check_layout(self, layout: LotLayout) -> None:
|
| 102 |
+
missing = sorted({rect.extent for rect in layout.rects} - set(self.extents))
|
| 103 |
+
if missing:
|
| 104 |
+
raise KeyError(f"layout uses extents with no basis: {missing}")
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
class LotVisualAdapter(nn.Module):
|
| 108 |
+
"""Extent heads and shape MLP wrapped around an external backbone."""
|
| 109 |
+
|
| 110 |
+
def __init__(
|
| 111 |
+
self,
|
| 112 |
+
token_dim: int,
|
| 113 |
+
hidden_size: int,
|
| 114 |
+
extents: list[tuple[int, int, int]],
|
| 115 |
+
*,
|
| 116 |
+
include_time: bool = False,
|
| 117 |
+
shape_hidden: int = 256,
|
| 118 |
+
):
|
| 119 |
+
super().__init__()
|
| 120 |
+
self.token_dim = int(token_dim)
|
| 121 |
+
self.hidden_size = int(hidden_size)
|
| 122 |
+
self.include_time = bool(include_time)
|
| 123 |
+
self.bank = ExtentBank(self.token_dim, extents)
|
| 124 |
+
feat_dim = 5 if self.include_time else 4
|
| 125 |
+
width = max(feat_dim, int(shape_hidden))
|
| 126 |
+
self.shape_mlp = nn.Sequential(
|
| 127 |
+
nn.Linear(feat_dim, width),
|
| 128 |
+
nn.SiLU(),
|
| 129 |
+
nn.Linear(width, self.hidden_size),
|
| 130 |
+
)
|
| 131 |
+
nn.init.zeros_(self.shape_mlp[-1].weight)
|
| 132 |
+
nn.init.zeros_(self.shape_mlp[-1].bias)
|
| 133 |
+
self.in_proj = nn.ModuleDict()
|
| 134 |
+
self.out_proj = nn.ModuleDict()
|
| 135 |
+
for extent in self.bank.extents:
|
| 136 |
+
dense_dim = self.bank.dense_dim(extent)
|
| 137 |
+
self.in_proj[_key(extent)] = nn.Linear(dense_dim, self.hidden_size)
|
| 138 |
+
self.out_proj[_key(extent)] = nn.Linear(self.hidden_size, dense_dim)
|
| 139 |
+
self.register_buffer("pretrained_in", torch.empty(self.hidden_size, self.token_dim))
|
| 140 |
+
self.register_buffer("pretrained_out", torch.empty(self.token_dim, self.hidden_size))
|
| 141 |
+
self.register_buffer("pretrained_bias_in", torch.empty(self.hidden_size))
|
| 142 |
+
self.register_buffer("pretrained_bias_out", torch.empty(self.token_dim))
|
| 143 |
+
self.register_buffer("pretrained_ready", torch.zeros((), dtype=torch.int32))
|
| 144 |
+
|
| 145 |
+
@torch.no_grad()
|
| 146 |
+
def init_from_pretrained(
|
| 147 |
+
self,
|
| 148 |
+
weight_in: torch.Tensor,
|
| 149 |
+
weight_out: torch.Tensor,
|
| 150 |
+
bias_in: torch.Tensor | None = None,
|
| 151 |
+
bias_out: torch.Tensor | None = None,
|
| 152 |
+
) -> None:
|
| 153 |
+
"""Initialize heads as ``W_in,e = W_in A^T`` and ``W_e = A W_out``.
|
| 154 |
+
|
| 155 |
+
``weight_in`` is ``(hidden, D)`` and ``weight_out`` is ``(D, hidden)``,
|
| 156 |
+
matching ``nn.Linear`` layout. Output bias is lifted by ``A`` so a
|
| 157 |
+
unit extent reproduces the pretrained bias and a larger extent starts
|
| 158 |
+
on the same subspace.
|
| 159 |
+
"""
|
| 160 |
+
hidden, token_dim = self.hidden_size, self.token_dim
|
| 161 |
+
if tuple(weight_in.shape) != (hidden, token_dim):
|
| 162 |
+
raise ValueError(f"weight_in shape {tuple(weight_in.shape)} != {(hidden, token_dim)}")
|
| 163 |
+
if tuple(weight_out.shape) != (token_dim, hidden):
|
| 164 |
+
raise ValueError(f"weight_out shape {tuple(weight_out.shape)} != {(token_dim, hidden)}")
|
| 165 |
+
self.pretrained_in.copy_(weight_in.detach().to(dtype=self.pretrained_in.dtype, device=self.pretrained_in.device).clone())
|
| 166 |
+
self.pretrained_out.copy_(weight_out.detach().to(dtype=self.pretrained_out.dtype, device=self.pretrained_out.device).clone())
|
| 167 |
+
if bias_in is None:
|
| 168 |
+
self.pretrained_bias_in.zero_()
|
| 169 |
+
else:
|
| 170 |
+
self.pretrained_bias_in.copy_(bias_in.detach().to(dtype=self.pretrained_bias_in.dtype, device=self.pretrained_bias_in.device).clone())
|
| 171 |
+
if bias_out is None:
|
| 172 |
+
self.pretrained_bias_out.zero_()
|
| 173 |
+
else:
|
| 174 |
+
self.pretrained_bias_out.copy_(bias_out.detach().to(dtype=self.pretrained_bias_out.dtype, device=self.pretrained_bias_out.device).clone())
|
| 175 |
+
self.pretrained_ready.fill_(1)
|
| 176 |
+
for extent in self.bank.extents:
|
| 177 |
+
key = _key(extent)
|
| 178 |
+
in_layer = self.in_proj[key]
|
| 179 |
+
out_layer = self.out_proj[key]
|
| 180 |
+
basis = self.bank.basis(extent).to(dtype=in_layer.weight.dtype, device=in_layer.weight.device)
|
| 181 |
+
weight_in_m = weight_in.to(dtype=in_layer.weight.dtype, device=in_layer.weight.device)
|
| 182 |
+
weight_out_m = weight_out.to(dtype=out_layer.weight.dtype, device=out_layer.weight.device)
|
| 183 |
+
in_layer.weight.copy_(weight_in_m @ basis.T)
|
| 184 |
+
out_layer.weight.copy_(basis @ weight_out_m)
|
| 185 |
+
if bias_in is None:
|
| 186 |
+
in_layer.bias.zero_()
|
| 187 |
+
else:
|
| 188 |
+
in_layer.bias.copy_(bias_in.to(dtype=in_layer.bias.dtype, device=in_layer.bias.device))
|
| 189 |
+
if bias_out is None:
|
| 190 |
+
out_layer.bias.zero_()
|
| 191 |
+
else:
|
| 192 |
+
lifted = basis @ bias_out.to(dtype=basis.dtype, device=basis.device)
|
| 193 |
+
out_layer.bias.copy_(lifted.to(dtype=out_layer.bias.dtype))
|
| 194 |
+
|
| 195 |
+
@torch.no_grad()
|
| 196 |
+
def fit_extent(self, extent: tuple[int, int, int], dense: torch.Tensor, reference: torch.Tensor) -> None:
|
| 197 |
+
"""Fit one basis, then rebuild every head from the stored pretrained maps.
|
| 198 |
+
|
| 199 |
+
``fit_`` on the bank alone updates ``A`` and leaves the heads on the
|
| 200 |
+
previous subspace. Call this instead once ``init_from_pretrained`` has run.
|
| 201 |
+
"""
|
| 202 |
+
self.bank.fit_(extent, dense, reference)
|
| 203 |
+
if int(self.pretrained_ready) != 1:
|
| 204 |
+
return
|
| 205 |
+
self.init_from_pretrained(
|
| 206 |
+
self.pretrained_in,
|
| 207 |
+
self.pretrained_out,
|
| 208 |
+
self.pretrained_bias_in,
|
| 209 |
+
self.pretrained_bias_out,
|
| 210 |
+
)
|
| 211 |
+
|
| 212 |
+
def embed(self, tokens: torch.Tensor, layout: LotLayout) -> tuple[torch.Tensor, torch.Tensor]:
|
| 213 |
+
"""Return ``(B, L, hidden)`` and ``(L, 3)`` finest-grid centers."""
|
| 214 |
+
self._check_tokens(tokens, layout)
|
| 215 |
+
hidden = tokens.new_zeros(tokens.shape[0], layout.count, self.hidden_size)
|
| 216 |
+
for extent, indices in layout.groups():
|
| 217 |
+
rects = [layout.rects[index] for index in indices]
|
| 218 |
+
dense = gather_extent(tokens, rects)
|
| 219 |
+
projected = self.in_proj[_key(extent)](dense)
|
| 220 |
+
features = shape_features(extent, self.include_time).to(device=tokens.device, dtype=tokens.dtype)
|
| 221 |
+
projected = projected + self.shape_mlp(features)
|
| 222 |
+
hidden[:, indices] = projected
|
| 223 |
+
centers = layout.centers().to(device=tokens.device)
|
| 224 |
+
return hidden, centers
|
| 225 |
+
|
| 226 |
+
def velocity_from_states(
|
| 227 |
+
self,
|
| 228 |
+
states: torch.Tensor,
|
| 229 |
+
tokens: torch.Tensor,
|
| 230 |
+
sigma: torch.Tensor | float,
|
| 231 |
+
layout: LotLayout,
|
| 232 |
+
*,
|
| 233 |
+
x0_minus_eps: bool = False,
|
| 234 |
+
) -> torch.Tensor:
|
| 235 |
+
"""Extent heads plus eq. 9, scattered onto the uniform lattice.
|
| 236 |
+
|
| 237 |
+
Eq. 9 assumes the head predicts ``P eps - x0`` and returns ``eps - x0``.
|
| 238 |
+
H3's head predicts the negation (``x0 - eps``; H3's scheduler steps ``x0 = x_t + sigma * v``).
|
| 239 |
+
``x0_minus_eps=True`` flips the head before recovery and the velocity
|
| 240 |
+
after, so the result stays in the caller's convention. A unit extent has
|
| 241 |
+
no complement, so 1x1 parity cannot see a wrong sign here.
|
| 242 |
+
"""
|
| 243 |
+
sign = -1.0 if x0_minus_eps else 1.0
|
| 244 |
+
self._check_tokens(tokens, layout)
|
| 245 |
+
if states.shape != (tokens.shape[0], layout.count, self.hidden_size):
|
| 246 |
+
raise ValueError(
|
| 247 |
+
f"backbone returned {tuple(states.shape)}, expected "
|
| 248 |
+
f"{(tokens.shape[0], layout.count, self.hidden_size)}"
|
| 249 |
+
)
|
| 250 |
+
velocity = tokens.new_zeros(tokens.shape)
|
| 251 |
+
for extent, indices in layout.groups():
|
| 252 |
+
rects = [layout.rects[index] for index in indices]
|
| 253 |
+
u_a = sign * self.out_proj[_key(extent)](states[:, indices])
|
| 254 |
+
x_t = gather_extent(tokens, rects)
|
| 255 |
+
recovered = recover_dense_velocity(u_a, x_t, self.bank.basis(extent), sigma)
|
| 256 |
+
scatter_extent(velocity, sign * recovered, rects)
|
| 257 |
+
return velocity
|
| 258 |
+
|
| 259 |
+
def forward(self, tokens, sigma, layout: LotLayout, backbone, backbone_kwargs=None):
|
| 260 |
+
"""Dense y-space velocity. ``backbone(hidden, centers, sigma, **kwargs)``."""
|
| 261 |
+
hidden, centers = self.embed(tokens, layout)
|
| 262 |
+
kwargs = {} if backbone_kwargs is None else backbone_kwargs
|
| 263 |
+
states = backbone(hidden, centers, sigma, **kwargs)
|
| 264 |
+
return self.velocity_from_states(states, tokens, sigma, layout)
|
| 265 |
+
|
| 266 |
+
def asymmetric_targets(
|
| 267 |
+
self,
|
| 268 |
+
y0: torch.Tensor,
|
| 269 |
+
noise: torch.Tensor,
|
| 270 |
+
layout: LotLayout,
|
| 271 |
+
) -> list[tuple[list[int], torch.Tensor]]:
|
| 272 |
+
"""Per-extent eq. 8 targets, for tests and for a direct head loss."""
|
| 273 |
+
self.bank._check_layout(layout)
|
| 274 |
+
targets = []
|
| 275 |
+
for extent, indices in layout.groups():
|
| 276 |
+
rects = [layout.rects[index] for index in indices]
|
| 277 |
+
clean = gather_extent(y0, rects)
|
| 278 |
+
eps = gather_extent(noise, rects)
|
| 279 |
+
targets.append((indices, asymmetric_target(clean, eps, self.bank.basis(extent))))
|
| 280 |
+
return targets
|
| 281 |
+
|
| 282 |
+
def _check_tokens(self, tokens: torch.Tensor, layout: LotLayout) -> None:
|
| 283 |
+
expected = (layout.time, layout.height, layout.width, self.token_dim)
|
| 284 |
+
if tokens.ndim != 5 or tuple(tokens.shape[1:]) != expected:
|
| 285 |
+
raise ValueError(f"tokens shape {tuple(tokens.shape)} != (B, {expected[0]}, {expected[1]}, {expected[2]}, {expected[3]})")
|
| 286 |
+
self.bank._check_layout(layout)
|
lot/flow.py
ADDED
|
@@ -0,0 +1,314 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Patch-wise asymmetric flow for Level-of-Token Diffusion.
|
| 2 |
+
|
| 3 |
+
The velocity algebra is LakonLab's ``AsymFlowMixin.asymflow_velocity`` with
|
| 4 |
+
scale ``s = 1`` and calibration ``k = 1``. That special case is equation 9 of
|
| 5 |
+
Kiyohiro Nakayama, Brian Chao, Jan Ackermann, Hansheng Chen, Federico Tombari, Leonidas Guibas, Lior Yariv and Gordon Wetzstein, 2026. LoT does not use AsymFlow's per-extent timestep
|
| 6 |
+
calibration (eq. 18). Extent scales are applied to the clean latent instead
|
| 7 |
+
(eq. 16), which keeps a single timestep for every token.
|
| 8 |
+
|
| 9 |
+
LakonLab is loaded from the ``common.py`` file directly. Importing the
|
| 10 |
+
``lakonlab`` package pulls in ``mmcv``, which this path does not need.
|
| 11 |
+
"""
|
| 12 |
+
|
| 13 |
+
from __future__ import annotations
|
| 14 |
+
|
| 15 |
+
import importlib.util
|
| 16 |
+
import os
|
| 17 |
+
from functools import lru_cache
|
| 18 |
+
from pathlib import Path
|
| 19 |
+
|
| 20 |
+
import torch
|
| 21 |
+
|
| 22 |
+
from layout import LotLayout, TokenRect
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def _lakonlab_common_path() -> Path:
|
| 26 |
+
override = os.environ.get("LAKONLAB_ROOT")
|
| 27 |
+
candidates: list[Path] = []
|
| 28 |
+
if override:
|
| 29 |
+
candidates.append(Path(override))
|
| 30 |
+
repo = Path(__file__).resolve().parents[2]
|
| 31 |
+
candidates.append(repo.parent / "LakonLab")
|
| 32 |
+
for root in candidates:
|
| 33 |
+
path = root / "lakonlab" / "models" / "architectures" / "asymflow" / "common.py"
|
| 34 |
+
if path.is_file():
|
| 35 |
+
return path
|
| 36 |
+
searched = ", ".join(str(path) for path in candidates)
|
| 37 |
+
raise FileNotFoundError(f"LakonLab asymflow/common.py not found. Looked in: {searched}")
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
@lru_cache(maxsize=1)
|
| 41 |
+
def _asymflow():
|
| 42 |
+
path = _lakonlab_common_path()
|
| 43 |
+
spec = importlib.util.spec_from_file_location("lakonlab_asymflow_common", path)
|
| 44 |
+
if spec is None or spec.loader is None:
|
| 45 |
+
raise ImportError(f"could not load {path}")
|
| 46 |
+
module = importlib.util.module_from_spec(spec)
|
| 47 |
+
spec.loader.exec_module(module)
|
| 48 |
+
|
| 49 |
+
class _VelocityKernel(module.AsymFlowMixin):
|
| 50 |
+
"""One Procrustes basis, otherwise the LakonLab velocity step."""
|
| 51 |
+
|
| 52 |
+
def __init__(self, basis: torch.Tensor):
|
| 53 |
+
self.proj_buffer = basis
|
| 54 |
+
self.sigma_min = 1e-6
|
| 55 |
+
self.training = True
|
| 56 |
+
|
| 57 |
+
return module.AsymFlowCalibration, _VelocityKernel, path
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def mean_basis(token_dim: int, extent: tuple[int, int, int]) -> torch.Tensor:
|
| 61 |
+
"""Semi-orthonormal lift that averages the uniform tokens inside a patch.
|
| 62 |
+
|
| 63 |
+
``A`` has shape ``(et*eh*ew*D, D)`` and ``A^T A = I``. For a unit extent
|
| 64 |
+
this is the identity, so a dense layout is unchanged. This is the
|
| 65 |
+
stand-in used before a Procrustes fit; it is not the paper's data-fit basis.
|
| 66 |
+
"""
|
| 67 |
+
et, eh, ew = extent
|
| 68 |
+
sites = et * eh * ew
|
| 69 |
+
if token_dim < 1 or sites < 1:
|
| 70 |
+
raise ValueError("token dim and extent must be positive")
|
| 71 |
+
eye = torch.eye(token_dim)
|
| 72 |
+
return eye.repeat(sites, 1) / math_sqrt(sites)
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def math_sqrt(value: int) -> float:
|
| 76 |
+
return value ** 0.5
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
def fit_procrustes(dense: torch.Tensor, reference: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
| 80 |
+
"""Orthogonal Procrustes basis and RMS scale, eqs. 14–15.
|
| 81 |
+
|
| 82 |
+
``dense`` is ``(N, D_e)`` and ``reference`` is ``(N, D)``, the aligned
|
| 83 |
+
multi-scale VAE tokens. The SVD matches
|
| 84 |
+
``LakonLab/tools/asymflow_subspace_procrustes.py``: ``A = V U^T`` from
|
| 85 |
+
``Z^T X = U S V^T``. ``A`` is ``(D_e, D)`` with ``A^T A = I``, and ``s``
|
| 86 |
+
matches the RMS of ``A^T x`` to the RMS of ``z``.
|
| 87 |
+
"""
|
| 88 |
+
if dense.ndim != 2 or reference.ndim != 2 or dense.shape[0] != reference.shape[0]:
|
| 89 |
+
raise ValueError("dense and reference must be (N, D_e) and (N, D)")
|
| 90 |
+
if reference.shape[1] > dense.shape[1]:
|
| 91 |
+
raise ValueError("reference dimension exceeds the dense patch dimension")
|
| 92 |
+
dense64 = dense.detach().double()
|
| 93 |
+
reference64 = reference.detach().double()
|
| 94 |
+
cross = reference64.T @ dense64
|
| 95 |
+
pixel_gram = dense64.T @ dense64
|
| 96 |
+
latent_norm_sq = (reference64 * reference64).sum().clamp(min=1e-12)
|
| 97 |
+
u, _, vh = torch.linalg.svd(cross, full_matrices=False)
|
| 98 |
+
rank = cross.shape[0]
|
| 99 |
+
basis = (vh[:rank].T @ u[:, :rank].T).contiguous()
|
| 100 |
+
projected = torch.trace(basis.T @ pixel_gram @ basis)
|
| 101 |
+
scale = (projected / latent_norm_sq).clamp(min=1e-12).sqrt()
|
| 102 |
+
return basis.float(), scale.float().reshape(())
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
def compress_patches(dense: torch.Tensor, basis: torch.Tensor) -> torch.Tensor:
|
| 106 |
+
"""Eq. 7. ``dense`` is ``(..., D_e)`` and the result is ``(..., D)``."""
|
| 107 |
+
return dense @ basis.to(dtype=dense.dtype, device=dense.device)
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
def asymmetric_target(clean: torch.Tensor, noise: torch.Tensor, basis: torch.Tensor) -> torch.Tensor:
|
| 111 |
+
"""Eq. 8, row-vector form: ``P eps - x0`` with ``P = A A^T``."""
|
| 112 |
+
projector = basis @ basis.T
|
| 113 |
+
return noise @ projector.to(dtype=noise.dtype, device=noise.device) - clean
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
def recover_dense_velocity(
|
| 117 |
+
u_a: torch.Tensor,
|
| 118 |
+
x_t: torch.Tensor,
|
| 119 |
+
basis: torch.Tensor,
|
| 120 |
+
sigma: torch.Tensor | float,
|
| 121 |
+
) -> torch.Tensor:
|
| 122 |
+
"""Recover a full-rank patch velocity from an asymmetric prediction (eq. 9).
|
| 123 |
+
|
| 124 |
+
``u_a`` and ``x_t`` share shape ``(..., D_e)``. ``sigma`` is a scalar or a
|
| 125 |
+
batch vector of length ``u_a.shape[0]``. The denominator is clamped at
|
| 126 |
+
``1e-6``, as in the paper.
|
| 127 |
+
"""
|
| 128 |
+
if u_a.shape != x_t.shape:
|
| 129 |
+
raise ValueError(f"u_a shape {tuple(u_a.shape)} != x_t shape {tuple(x_t.shape)}")
|
| 130 |
+
if basis.ndim != 2 or basis.shape[0] != u_a.shape[-1]:
|
| 131 |
+
raise ValueError("basis must be (D_e, D) for this patch")
|
| 132 |
+
sigma_t = _broadcast_sigma(sigma, u_a)
|
| 133 |
+
basis_f = basis.detach().to(device=u_a.device, dtype=torch.float32)
|
| 134 |
+
if os.environ.get("LOT_NO_LAKONLAB") == "1":
|
| 135 |
+
return asymflow_velocity_torch(u_a, x_t, basis_f, sigma_t).to(dtype=u_a.dtype)
|
| 136 |
+
try:
|
| 137 |
+
calibration_cls, kernel_cls, _path = _asymflow()
|
| 138 |
+
except FileNotFoundError:
|
| 139 |
+
# No LakonLab checkout (for example inside ComfyUI): the same s = k = 1 arithmetic.
|
| 140 |
+
return asymflow_velocity_torch(u_a, x_t, basis_f, sigma_t).to(dtype=u_a.dtype)
|
| 141 |
+
ones = torch.ones((), device=u_a.device, dtype=torch.float32)
|
| 142 |
+
calibration = calibration_cls(s=ones, k=ones, timestep=ones, sigma=sigma_t)
|
| 143 |
+
recovered = kernel_cls(basis_f).asymflow_velocity(u_a, x_t, calibration)
|
| 144 |
+
return recovered.to(dtype=u_a.dtype)
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
def asymflow_velocity_torch(
|
| 148 |
+
u_a: torch.Tensor,
|
| 149 |
+
x_t: torch.Tensor,
|
| 150 |
+
basis: torch.Tensor,
|
| 151 |
+
sigma: torch.Tensor,
|
| 152 |
+
sigma_min: float = 1e-6,
|
| 153 |
+
) -> torch.Tensor:
|
| 154 |
+
"""LakonLab ``asymflow_velocity`` with ``s = k = 1``, in plain torch (eq. 9).
|
| 155 |
+
|
| 156 |
+
``sub(v) = v A A^T`` and ``comp(v) = v - sub(v)``; the result is
|
| 157 |
+
``sub(u_a) + (comp(x_t) + comp(u_a)) / max(sigma, sigma_min)``. Matches the
|
| 158 |
+
kernel's training-mode ``sigma_min``, which is what ``_VelocityKernel`` sets.
|
| 159 |
+
"""
|
| 160 |
+
u = u_a.float()
|
| 161 |
+
x = x_t.float()
|
| 162 |
+
projector = basis @ basis.T
|
| 163 |
+
u_sub = u @ projector
|
| 164 |
+
x_sub = x @ projector
|
| 165 |
+
sig = sigma.float().clamp(min=sigma_min)
|
| 166 |
+
return u_sub + ((x - x_sub) + (u - u_sub)) / sig
|
| 167 |
+
|
| 168 |
+
|
| 169 |
+
def _broadcast_sigma(sigma: torch.Tensor | float, like: torch.Tensor) -> torch.Tensor:
|
| 170 |
+
if not torch.is_tensor(sigma):
|
| 171 |
+
sigma = torch.tensor(float(sigma), dtype=torch.float32)
|
| 172 |
+
sigma = sigma.to(device=like.device, dtype=torch.float32)
|
| 173 |
+
if sigma.ndim == 0:
|
| 174 |
+
return sigma
|
| 175 |
+
if sigma.ndim == 1:
|
| 176 |
+
if sigma.shape[0] != like.shape[0]:
|
| 177 |
+
raise ValueError(f"sigma batch {sigma.shape[0]} != {like.shape[0]}")
|
| 178 |
+
return sigma.reshape(like.shape[0], *([1] * (like.ndim - 1)))
|
| 179 |
+
if sigma.shape == like.shape[: sigma.ndim]:
|
| 180 |
+
return sigma.reshape(*sigma.shape, *([1] * (like.ndim - sigma.ndim)))
|
| 181 |
+
raise ValueError(f"cannot broadcast sigma shape {tuple(sigma.shape)} onto {tuple(like.shape)}")
|
| 182 |
+
|
| 183 |
+
|
| 184 |
+
def sample_noisy(
|
| 185 |
+
y0: torch.Tensor,
|
| 186 |
+
sigma: torch.Tensor | float,
|
| 187 |
+
noise: torch.Tensor | None = None,
|
| 188 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 189 |
+
"""``y_t = (1 - sigma) y0 + sigma eps`` on the scaled lattice."""
|
| 190 |
+
if noise is None:
|
| 191 |
+
noise = torch.randn_like(y0)
|
| 192 |
+
sig = _broadcast_sigma(sigma, y0).to(dtype=y0.dtype)
|
| 193 |
+
return (1 - sig) * y0 + sig * noise, noise
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
def clean_from_velocity(
|
| 197 |
+
y_t: torch.Tensor,
|
| 198 |
+
velocity: torch.Tensor,
|
| 199 |
+
sigma: torch.Tensor | float,
|
| 200 |
+
) -> torch.Tensor:
|
| 201 |
+
"""``y0_hat = y_t - sigma u``."""
|
| 202 |
+
sig = _broadcast_sigma(sigma, y_t).to(dtype=y_t.dtype)
|
| 203 |
+
return y_t - sig * velocity
|
| 204 |
+
|
| 205 |
+
|
| 206 |
+
def lot_clean_loss(
|
| 207 |
+
y0_hat: torch.Tensor,
|
| 208 |
+
y0: torch.Tensor,
|
| 209 |
+
sigma: torch.Tensor | float,
|
| 210 |
+
sigma_floor: float = 0.05,
|
| 211 |
+
) -> torch.Tensor:
|
| 212 |
+
"""Weighted clean-data loss, eq. 17.
|
| 213 |
+
|
| 214 |
+
The paper writes a squared Euclidean norm inside the expectation. This
|
| 215 |
+
returns the mean over elements of that weighted square, which has the
|
| 216 |
+
same minimizer on a fixed grid and does not grow with the lattice size.
|
| 217 |
+
"""
|
| 218 |
+
sig = _broadcast_sigma(sigma, y0).clamp(min=sigma_floor)
|
| 219 |
+
weight = 1.0 / sig.square()
|
| 220 |
+
return ((y0_hat - y0).float().square() * weight).mean()
|
| 221 |
+
|
| 222 |
+
|
| 223 |
+
|
| 224 |
+
def lot_h3_clean_loss(
|
| 225 |
+
out: torch.Tensor,
|
| 226 |
+
y_t: torch.Tensor,
|
| 227 |
+
y0: torch.Tensor,
|
| 228 |
+
sigma: torch.Tensor | float,
|
| 229 |
+
sigma_floor: float = 0.05,
|
| 230 |
+
) -> torch.Tensor:
|
| 231 |
+
"""Eq. 17 for H3's head, which predicts ``x0 - eps`` (H3's scheduler: ``x0 = x_t + sigma * v``).
|
| 232 |
+
|
| 233 |
+
The clean estimate is ``y_t + sigma * out``, not ``clean_from_velocity`` (that
|
| 234 |
+
is the ``eps - x0`` convention). For ``sigma >= sigma_floor`` this equals
|
| 235 |
+
``mean((out - (y0 - eps))^2)``: the ordinary H3 velocity MSE, taken in y-space.
|
| 236 |
+
"""
|
| 237 |
+
sig = _broadcast_sigma(sigma, y0).to(dtype=torch.float32)
|
| 238 |
+
y0_hat = y_t.float() + sig * out.float()
|
| 239 |
+
weight = 1.0 / sig.clamp(min=sigma_floor).square()
|
| 240 |
+
return ((y0_hat - y0.float()).square() * weight).mean()
|
| 241 |
+
|
| 242 |
+
def _one_extent(rects: list[TokenRect]) -> tuple[int, int, int]:
|
| 243 |
+
if not rects:
|
| 244 |
+
raise ValueError("expected at least one rectangle")
|
| 245 |
+
extent = rects[0].extent
|
| 246 |
+
for rect in rects[1:]:
|
| 247 |
+
if rect.extent != extent:
|
| 248 |
+
raise ValueError(f"rectangles mix extents {extent} and {rect.extent}")
|
| 249 |
+
return extent
|
| 250 |
+
|
| 251 |
+
|
| 252 |
+
def _windows(rects: list[TokenRect], device: torch.device) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
| 253 |
+
"""Broadcast indices ``(n, et, eh, ew)`` for every rectangle of one extent."""
|
| 254 |
+
et, eh, ew = _one_extent(rects)
|
| 255 |
+
t = torch.tensor([rect.t for rect in rects], dtype=torch.long, device=device)
|
| 256 |
+
u = torch.tensor([rect.u for rect in rects], dtype=torch.long, device=device)
|
| 257 |
+
v = torch.tensor([rect.v for rect in rects], dtype=torch.long, device=device)
|
| 258 |
+
dt = torch.arange(et, device=device)
|
| 259 |
+
du = torch.arange(eh, device=device)
|
| 260 |
+
dv = torch.arange(ew, device=device)
|
| 261 |
+
tt = t[:, None, None, None] + dt[None, :, None, None]
|
| 262 |
+
uu = u[:, None, None, None] + du[None, None, :, None]
|
| 263 |
+
vv = v[:, None, None, None] + dv[None, None, None, :]
|
| 264 |
+
return tt, uu, vv
|
| 265 |
+
|
| 266 |
+
|
| 267 |
+
def apply_extent_scales(
|
| 268 |
+
latent: torch.Tensor,
|
| 269 |
+
layout: LotLayout,
|
| 270 |
+
scales: dict[tuple[int, int, int], torch.Tensor],
|
| 271 |
+
*,
|
| 272 |
+
invert: bool = False,
|
| 273 |
+
) -> torch.Tensor:
|
| 274 |
+
"""Divide each patch by its extent scale. ``invert=True`` multiplies it back.
|
| 275 |
+
|
| 276 |
+
One index write per extent. A unit scale hides a swapped multiply, so tests
|
| 277 |
+
use a scale other than 1.
|
| 278 |
+
"""
|
| 279 |
+
out = latent.clone()
|
| 280 |
+
buckets: dict[tuple[int, int, int], list[TokenRect]] = {}
|
| 281 |
+
for rect in layout.rects:
|
| 282 |
+
buckets.setdefault(rect.extent, []).append(rect)
|
| 283 |
+
for extent, rects in buckets.items():
|
| 284 |
+
scale = scales[extent].to(device=latent.device, dtype=latent.dtype)
|
| 285 |
+
tt, uu, vv = _windows(rects, latent.device)
|
| 286 |
+
region = out[:, tt, uu, vv, :]
|
| 287 |
+
region = region * scale if invert else region / scale
|
| 288 |
+
out[:, tt, uu, vv, :] = region
|
| 289 |
+
return out
|
| 290 |
+
|
| 291 |
+
|
| 292 |
+
def gather_extent(tokens: torch.Tensor, rects: list[TokenRect]) -> torch.Tensor:
|
| 293 |
+
"""Pack patches of one extent to ``(B, n, D_e)``.
|
| 294 |
+
|
| 295 |
+
Site order is C-order over ``(et, eh, ew, D)``, with the token channel
|
| 296 |
+
fastest. Procrustes rows must use this same order. One index read for the
|
| 297 |
+
whole extent.
|
| 298 |
+
"""
|
| 299 |
+
tt, uu, vv = _windows(rects, tokens.device)
|
| 300 |
+
patches = tokens[:, tt, uu, vv, :]
|
| 301 |
+
return patches.reshape(tokens.shape[0], len(rects), -1)
|
| 302 |
+
|
| 303 |
+
|
| 304 |
+
def scatter_extent(canvas: torch.Tensor, values: torch.Tensor, rects: list[TokenRect]) -> None:
|
| 305 |
+
"""Write ``(B, n, D_e)`` patches back onto a fresh ``(B, T, H, W, D)`` canvas."""
|
| 306 |
+
tt, uu, vv = _windows(rects, canvas.device)
|
| 307 |
+
et, eh, ew = _one_extent(rects)
|
| 308 |
+
patches = values.reshape(canvas.shape[0], len(rects), et, eh, ew, canvas.shape[-1])
|
| 309 |
+
canvas[:, tt, uu, vv, :] = patches
|
| 310 |
+
|
| 311 |
+
|
| 312 |
+
def euler_step(y_t: torch.Tensor, velocity: torch.Tensor, t: float, t_next: float) -> torch.Tensor:
|
| 313 |
+
"""One explicit step of ``dy/dt = u`` from ``t`` to ``t_next``."""
|
| 314 |
+
return y_t + (float(t_next) - float(t)) * velocity
|
lot/gpu_guard.py
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""One H3 GPU job at a time. The guard refuses; it never kills anything."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import math
|
| 6 |
+
import os
|
| 7 |
+
import subprocess
|
| 8 |
+
|
| 9 |
+
GPU_SCRIPTS = ("train_synth.py", "parity_h3.py", "procrustes_h3.py", "make_pairs_h3.py", "render_h3.py", "time_decode_h3.py", "train_h3.py", "build_cache_h3.py", "time_lot_h3.py", "mixed_res_h3.py", "two_track_h3.py", "train_tt.py", "build_pairs_tt.py")
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def other_gpu_job() -> str | None:
|
| 13 |
+
"""Command line of another LoT GPU script on this machine, or ``None``."""
|
| 14 |
+
listing = subprocess.run(["ps", "-eo", "pid=,args="], capture_output=True, text=True)
|
| 15 |
+
me = os.getpid()
|
| 16 |
+
for line in listing.stdout.splitlines():
|
| 17 |
+
pid, _, cmd = line.strip().partition(" ")
|
| 18 |
+
argv = cmd.split()
|
| 19 |
+
if not pid.isdigit() or int(pid) == me or not argv:
|
| 20 |
+
continue
|
| 21 |
+
# argv[0] must be the interpreter: a shell whose command string names a
|
| 22 |
+
# script is a wrapper, not the job.
|
| 23 |
+
if not os.path.basename(argv[0]).startswith("python"):
|
| 24 |
+
continue
|
| 25 |
+
if any(os.path.basename(arg) in GPU_SCRIPTS for arg in argv[1:]):
|
| 26 |
+
return cmd
|
| 27 |
+
return None
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def refuse_if_busy(name: str) -> None:
|
| 31 |
+
job = other_gpu_job()
|
| 32 |
+
if job is not None:
|
| 33 |
+
raise SystemExit(f"{name}: another LoT GPU job is running ({job}). It will not be killed.")
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def card_gb() -> float:
|
| 37 |
+
"""Free memory on the current CUDA device, GB (other processes count against it)."""
|
| 38 |
+
import torch
|
| 39 |
+
|
| 40 |
+
free, _total = torch.cuda.mem_get_info()
|
| 41 |
+
return free / 2**30
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def auto_swap(value: str, inference: bool = False, latent_t: int = 7) -> int:
|
| 45 |
+
"""``--swap auto``: DiT blocks to stream from CPU, from the card's free memory.
|
| 46 |
+
|
| 47 |
+
Measured with ``train_tt.py lora --check`` (two-track LoRA, 1344x768 pairs): 8 blocks peak at
|
| 48 |
+
19.6 GB on an RTX 5090; 24 blocks peak at 13.3 GB on an RTX PRO 4000 with 6 GB already taken.
|
| 49 |
+
0 (all resident) on 64 GB+ free, e.g. an RTX PRO 6000 (96 GB), is not measured yet."""
|
| 50 |
+
if value != "auto":
|
| 51 |
+
return int(value)
|
| 52 |
+
gb = card_gb()
|
| 53 |
+
if inference:
|
| 54 |
+
# Rendering (no LoRA grads, no checkpointing), measured 2026-11-13 on an RTX 5090 at 1344x768, 22 frames,
|
| 55 |
+
# VAE decode included: peak 26.8 GB with every block resident, about 0.42 GB less per streamed block.
|
| 56 |
+
# Streaming costs two-track the most (LR steps are short): 14.0 s at 0-2 streamed, 20.9 s at 8.
|
| 57 |
+
# Longer clips need more (activations, VAE decode): 22 latent frames (73 pixel frames) ran out of memory
|
| 58 |
+
# with none streamed and fit with 6. Assumed 0.35 GB per latent frame beyond 7.
|
| 59 |
+
need = 26.8 + 0.35 * max(0, latent_t - 7) + 1.0
|
| 60 |
+
return 0 if gb >= need else min(50, int(math.ceil((need - gb) / 0.42)))
|
| 61 |
+
return 0 if gb >= 64 else 8 if gb >= 28 else 16 if gb >= 21 else 24
|
lot/h3.py
ADDED
|
@@ -0,0 +1,180 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""MiniMax-H3 geometry for a Level-of-Token visual stream.
|
| 2 |
+
|
| 3 |
+
H3's native patch is ``(t, h, w) = (1, 2, 2)`` on 24-channel video latents,
|
| 4 |
+
so the uniform token dimension is ``24 * 1 * 2 * 2 = 96``. LoT extents are
|
| 5 |
+
counted in those tokens, not in latent pixels. A paper video extent of
|
| 6 |
+
``2 x 2`` therefore covers a ``1 x 4 x 4`` block of VAE latents.
|
| 7 |
+
|
| 8 |
+
The Wan training setup in Appendix A.10 keeps ``e_t = 1`` and draws spatial
|
| 9 |
+
extents from ``{1, 2, 4}``, including rectangles. That is the H3 preset.
|
| 10 |
+
"""
|
| 11 |
+
|
| 12 |
+
from __future__ import annotations
|
| 13 |
+
|
| 14 |
+
import itertools
|
| 15 |
+
|
| 16 |
+
import torch
|
| 17 |
+
|
| 18 |
+
from adapter import LotVisualAdapter
|
| 19 |
+
from layout import TokenRect, dense_layout, layout_from_rects
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
H3_LATENT_CHANNELS = 24
|
| 23 |
+
H3_PATCH = (1, 2, 2)
|
| 24 |
+
H3_TOKEN_DIM = H3_LATENT_CHANNELS * H3_PATCH[0] * H3_PATCH[1] * H3_PATCH[2]
|
| 25 |
+
H3_HIDDEN = 5376
|
| 26 |
+
H3_HEADS = 56
|
| 27 |
+
H3_HEAD_DIM = 128
|
| 28 |
+
|
| 29 |
+
# Appendix A.10. Temporal extent stays 1; spatial sides are 1, 2, or 4.
|
| 30 |
+
H3_SPATIAL_SIDES = (1, 2, 4)
|
| 31 |
+
H3_EXTENTS = tuple((1, eh, ew) for eh, ew in itertools.product(H3_SPATIAL_SIDES, repeat=2))
|
| 32 |
+
|
| 33 |
+
# Image quadtree in Appendix A.3. Squares only, up to 8.
|
| 34 |
+
IMAGE_EXTENTS = ((1, 1, 1), (1, 2, 2), (1, 4, 4), (1, 8, 8))
|
| 35 |
+
|
| 36 |
+
# Wan's per-frame token cap. H3 resolutions differ; this is a budget check,
|
| 37 |
+
# not a hard limit of the adapter.
|
| 38 |
+
WAN_MAX_TOKENS_PER_FRAME = 3072
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def patchify(latent: torch.Tensor, patch: tuple[int, int, int] = H3_PATCH) -> torch.Tensor:
|
| 42 |
+
"""``(B, C, T, H, W)`` -> ``(B, T', H', W', C*pt*ph*pw)``.
|
| 43 |
+
|
| 44 |
+
Channel order inside the token is C-order ``(channel, pt, ph, pw)`` with
|
| 45 |
+
``pw`` fastest.
|
| 46 |
+
"""
|
| 47 |
+
if latent.ndim != 5:
|
| 48 |
+
raise ValueError("latent must be (B, C, T, H, W)")
|
| 49 |
+
pt, ph, pw = patch
|
| 50 |
+
batch, channels, time, height, width = latent.shape
|
| 51 |
+
if time % pt or height % ph or width % pw:
|
| 52 |
+
raise ValueError(f"latent {(time, height, width)} is not divisible by patch {patch}")
|
| 53 |
+
tt, hh, ww = time // pt, height // ph, width // pw
|
| 54 |
+
tokens = latent.reshape(batch, channels, tt, pt, hh, ph, ww, pw)
|
| 55 |
+
tokens = tokens.permute(0, 2, 4, 6, 1, 3, 5, 7)
|
| 56 |
+
return tokens.reshape(batch, tt, hh, ww, channels * pt * ph * pw)
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def unpatchify(
|
| 60 |
+
tokens: torch.Tensor,
|
| 61 |
+
patch: tuple[int, int, int] = H3_PATCH,
|
| 62 |
+
channels: int = H3_LATENT_CHANNELS,
|
| 63 |
+
) -> torch.Tensor:
|
| 64 |
+
"""Inverse of ``patchify``."""
|
| 65 |
+
pt, ph, pw = patch
|
| 66 |
+
if tokens.ndim != 5 or tokens.shape[-1] != channels * pt * ph * pw:
|
| 67 |
+
raise ValueError("token shape does not match channels and patch")
|
| 68 |
+
batch, tt, hh, ww, _dense = tokens.shape
|
| 69 |
+
tokens = tokens.reshape(batch, tt, hh, ww, channels, pt, ph, pw)
|
| 70 |
+
tokens = tokens.permute(0, 4, 1, 5, 2, 6, 3, 7)
|
| 71 |
+
return tokens.reshape(batch, channels, tt * pt, hh * ph, ww * pw)
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def make_h3_adapter(
|
| 75 |
+
hidden_size: int = H3_HIDDEN,
|
| 76 |
+
*,
|
| 77 |
+
include_time: bool = False,
|
| 78 |
+
shape_hidden: int = 256,
|
| 79 |
+
extents: tuple[tuple[int, int, int], ...] = H3_EXTENTS,
|
| 80 |
+
) -> LotVisualAdapter:
|
| 81 |
+
"""Adapter whose token dim matches H3. Weights are not loaded here."""
|
| 82 |
+
return LotVisualAdapter(
|
| 83 |
+
H3_TOKEN_DIM,
|
| 84 |
+
hidden_size,
|
| 85 |
+
list(extents),
|
| 86 |
+
include_time=include_time,
|
| 87 |
+
shape_hidden=shape_hidden,
|
| 88 |
+
)
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
def tokens_per_frame(layout_count: int, time: int) -> float:
|
| 92 |
+
if time < 1:
|
| 93 |
+
raise ValueError("time must be positive")
|
| 94 |
+
return layout_count / time
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
def over_wan_frame_budget(layout_count: int, time: int) -> bool:
|
| 98 |
+
return tokens_per_frame(layout_count, time) > WAN_MAX_TOKENS_PER_FRAME
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
def gate_frame_layout(height: int = 24, width: int = 42):
|
| 102 |
+
"""One frame of three horizontal bands: 1×1, then 2×2, then 4×2.
|
| 103 |
+
|
| 104 |
+
The default is the 768×1344 timing clip: 24×42 tokens, 462 rectangles.
|
| 105 |
+
Width 42 is not divisible by 4, so the coarse band is 4×2, not 4×4.
|
| 106 |
+
Rows 0–7 are 1×1, rows 8–15 are 2×2, rows 16–23 are 4×2. The phase-4
|
| 107 |
+
smoke canvas (384×640) is ``gate_frame_layout(12, 20)``.
|
| 108 |
+
"""
|
| 109 |
+
band = height // 3
|
| 110 |
+
if height % 3 or band % 4 or width % 2:
|
| 111 |
+
raise ValueError(f"token grid {(height, width)} needs height a multiple of 12 and an even width")
|
| 112 |
+
rects = []
|
| 113 |
+
for u in range(band):
|
| 114 |
+
for v in range(width):
|
| 115 |
+
rects.append(TokenRect(0, u, v, 1, 1, 1))
|
| 116 |
+
for u in range(band, 2 * band, 2):
|
| 117 |
+
for v in range(0, width, 2):
|
| 118 |
+
rects.append(TokenRect(0, u, v, 1, 2, 2))
|
| 119 |
+
for u in range(2 * band, height, 4):
|
| 120 |
+
for v in range(0, width, 2):
|
| 121 |
+
rects.append(TokenRect(0, u, v, 1, 4, 2))
|
| 122 |
+
return layout_from_rects(1, height, width, rects)
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
def clip_layout(frames: int, height: int = 24, width: int = 42):
|
| 126 |
+
"""``gate_frame_layout`` repeated on every latent frame."""
|
| 127 |
+
frame = gate_frame_layout(height, width)
|
| 128 |
+
rects = [
|
| 129 |
+
TokenRect(t, rect.u, rect.v, 1, rect.eh, rect.ew)
|
| 130 |
+
for t in range(frames)
|
| 131 |
+
for rect in frame.rects
|
| 132 |
+
]
|
| 133 |
+
return layout_from_rects(frames, height, width, rects)
|
| 134 |
+
|
| 135 |
+
|
| 136 |
+
GRID_LAYOUTS = ("dense", "bands", "center", "uniform2", "uniform4")
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
def grid_layout(name: str, time: int, height: int, width: int):
|
| 140 |
+
"""A LoT layout for any token grid, in 4×4 super-cells; edge leftovers stay 1×1.
|
| 141 |
+
|
| 142 |
+
``bands``: top third 1×1, middle 2×2, bottom 4×2 (by super-cell row).
|
| 143 |
+
``center``: 1×1 near the middle, 2×2 around it, 4×4 at the edges; meant for
|
| 144 |
+
portraits and talking heads, where the face sits in the middle.
|
| 145 |
+
``uniformN``: every full super-cell N×N. ``dense``: all 1×1.
|
| 146 |
+
"""
|
| 147 |
+
if name not in GRID_LAYOUTS:
|
| 148 |
+
raise ValueError(f"layout {name!r} is not one of {GRID_LAYOUTS}")
|
| 149 |
+
if name == "dense":
|
| 150 |
+
return dense_layout(time, height, width) # raster order: bit-identical to the stock forward
|
| 151 |
+
rows, cols = height // 4, width // 4
|
| 152 |
+
|
| 153 |
+
def extent(cell_u: int, cell_v: int) -> tuple[int, int, int]:
|
| 154 |
+
if name == "dense":
|
| 155 |
+
return (1, 1, 1)
|
| 156 |
+
if name == "uniform2":
|
| 157 |
+
return (1, 2, 2)
|
| 158 |
+
if name == "uniform4":
|
| 159 |
+
return (1, 4, 4)
|
| 160 |
+
if name == "bands":
|
| 161 |
+
third = cell_u * 3 // max(rows, 1)
|
| 162 |
+
return ((1, 1, 1), (1, 2, 2), (1, 4, 2))[min(third, 2)]
|
| 163 |
+
du = (cell_u + 0.5) / max(rows, 1) - 0.5
|
| 164 |
+
dv = (cell_v + 0.5) / max(cols, 1) - 0.5
|
| 165 |
+
radius = (du * du + dv * dv) ** 0.5 / 0.7071
|
| 166 |
+
return (1, 1, 1) if radius < 0.35 else (1, 2, 2) if radius < 0.7 else (1, 4, 4)
|
| 167 |
+
|
| 168 |
+
rects = []
|
| 169 |
+
for t in range(time):
|
| 170 |
+
for u in range(height):
|
| 171 |
+
for v in range(width):
|
| 172 |
+
if u >= rows * 4 or v >= cols * 4:
|
| 173 |
+
rects.append(TokenRect(t, u, v, 1, 1, 1)) # edge leftovers
|
| 174 |
+
for cu in range(rows):
|
| 175 |
+
for cv in range(cols):
|
| 176 |
+
_et, eh, ew = extent(cu, cv)
|
| 177 |
+
for u in range(cu * 4, cu * 4 + 4, eh):
|
| 178 |
+
for v in range(cv * 4, cv * 4 + 4, ew):
|
| 179 |
+
rects.append(TokenRect(t, u, v, 1, eh, ew))
|
| 180 |
+
return layout_from_rects(time, height, width, rects)
|
lot/h3_musubi.py
ADDED
|
@@ -0,0 +1,356 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""H3 through musubi-tuner, with an optional Level-of-Token target-video tail.
|
| 2 |
+
|
| 3 |
+
musubi-tuner (https://github.com/kohya-ss/musubi-tuner, Apache-2.0) loads the
|
| 4 |
+
published ComfyUI ConvRot int8 checkpoints (pruned or full), streams blocks
|
| 5 |
+
from CPU, checkpoints activations, and trains LoRAs on the int8 base. This
|
| 6 |
+
module uses it as a library and changes none of its code. ``predict`` runs
|
| 7 |
+
the dense model, or, with ``lot=LotSplice(...)``, the same packed sequence
|
| 8 |
+
with three changes on the target video only:
|
| 9 |
+
|
| 10 |
+
1. its rows are embedded by the splice's extent heads (fewer, larger tokens),
|
| 11 |
+
2. their rotary positions are the rectangle centres,
|
| 12 |
+
3. the final norm + AdaLN feed the splice's heads instead of ``video_out``.
|
| 13 |
+
|
| 14 |
+
Optional, from MPEG-Diffusion (``mixed_res``): ``dual=DualKV(...)`` gives 1x1 queries a dense
|
| 15 |
+
key/value view with coarse keys expanded to their cells; ``capture={"block": i}`` returns that
|
| 16 |
+
block's video states (for the region head) and, with ``"kv": {}``, each block's pre-RoPE video
|
| 17 |
+
keys/values (for fitting the up-maps).
|
| 18 |
+
|
| 19 |
+
Text, audio, and the block stack are untouched. The output is H3's dense
|
| 20 |
+
``x0 - eps`` velocity in every case, so the samplers step it unchanged.
|
| 21 |
+
|
| 22 |
+
Audio: a still or clip trained without sound still packs H3's audio block,
|
| 23 |
+
because the base was trained with it. Those rows are silence (``x0 = 0``)
|
| 24 |
+
noised at the same schedule position on the audio shift (3) as the video
|
| 25 |
+
rows are on the video shift (12). They carry no loss.
|
| 26 |
+
"""
|
| 27 |
+
|
| 28 |
+
from __future__ import annotations
|
| 29 |
+
|
| 30 |
+
import contextlib
|
| 31 |
+
import sys
|
| 32 |
+
from dataclasses import replace
|
| 33 |
+
|
| 34 |
+
import os
|
| 35 |
+
import torch
|
| 36 |
+
|
| 37 |
+
from lot_paths import LOT_H3_CHECKPOINT, MUSUBI_SRC
|
| 38 |
+
|
| 39 |
+
if MUSUBI_SRC not in sys.path:
|
| 40 |
+
sys.path.insert(0, MUSUBI_SRC)
|
| 41 |
+
|
| 42 |
+
FRAME_PER_TOKEN = (1, 4, 4, 4, 4)
|
| 43 |
+
VIDEO_SHIFT = 12.0
|
| 44 |
+
AUDIO_SHIFT = 3.0
|
| 45 |
+
AUDIO_CHANNELS = 32
|
| 46 |
+
LORA_INCLUDE = r"blocks\.\d+\.(?:attn\.(?:qkv_proj|out_proj)|mlp\.(?:fc1|fc2))"
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
# ---------------------------------------------------------------------------- schedules
|
| 50 |
+
|
| 51 |
+
def shift_sigma(sigma, shift: float):
|
| 52 |
+
"""``shift * sigma / (1 + (shift - 1) * sigma)``."""
|
| 53 |
+
return shift * sigma / (1.0 + (shift - 1.0) * sigma)
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def unshift_sigma(sigma, shift: float):
|
| 57 |
+
"""Inverse of ``shift_sigma``."""
|
| 58 |
+
return sigma / (shift + sigma * (1.0 - shift))
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def audio_sigma(video_sigma):
|
| 62 |
+
"""The audio sigma at the same base-schedule position as ``video_sigma``."""
|
| 63 |
+
return shift_sigma(unshift_sigma(video_sigma, VIDEO_SHIFT), AUDIO_SHIFT)
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def sample_sigma(shift, device, generator: torch.Generator | None = None, count: int = 1) -> torch.Tensor:
|
| 67 |
+
"""``count`` training sigmas in (0, 1), shape ``(count,)``.
|
| 68 |
+
|
| 69 |
+
A float ``S`` maps a uniform base through the shift (12 is H3's own
|
| 70 |
+
density, the same as musubi's ``--h3_shift_video``). ``"sigmoid"`` is an
|
| 71 |
+
unshifted logit-normal. ``"lognorm:S"`` is a logit-normal base under shift ``S``.
|
| 72 |
+
"""
|
| 73 |
+
if shift == "sigmoid":
|
| 74 |
+
return torch.sigmoid(torch.randn(count, device=device, generator=generator))
|
| 75 |
+
if isinstance(shift, str) and shift.startswith("lognorm:"):
|
| 76 |
+
base = torch.sigmoid(torch.randn(count, device=device, generator=generator))
|
| 77 |
+
return shift_sigma(base, float(shift.split(":", 1)[1]))
|
| 78 |
+
return shift_sigma(torch.rand(count, device=device, generator=generator), float(shift))
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
def pixel_frames(latent_t: int) -> int:
|
| 82 |
+
"""Pixel frames a ``latent_t``-frame latent decodes to: 1, 5, 9, 13, 17, 22, ..."""
|
| 83 |
+
return sum(FRAME_PER_TOKEN[k % 5] for k in range(int(latent_t)))
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
def latent_frames(num_frames: int) -> int:
|
| 87 |
+
"""Pixel frames (snapped down onto 17n+5) -> latent frames; 1 is a still."""
|
| 88 |
+
if num_frames <= 1:
|
| 89 |
+
return 1
|
| 90 |
+
n = max(5, int(num_frames))
|
| 91 |
+
n -= (n - 5) % 17
|
| 92 |
+
return 5 * (n - 5) // 17 + 2
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
def audio_frames(latent_t: int) -> int:
|
| 96 |
+
"""40 Hz audio latents covering the clip, H3's ``(10 f + 3) // 6`` for ``f`` pixel frames."""
|
| 97 |
+
return (10 * pixel_frames(latent_t) + 3) // 6
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
# ------------------------------------------------------------------------------ loading
|
| 101 |
+
|
| 102 |
+
def rope_split_half(hidden_states: torch.Tensor, rotation_table: torch.Tensor) -> torch.Tensor:
|
| 103 |
+
"""musubi-tuner's ``_apply_rope_split_half`` written element-wise: same arithmetic, without the matmul on
|
| 104 |
+
a table broadcast over every head (which materialises ``L x heads x pairs x 2 x 2``)."""
|
| 105 |
+
pairs = rotation_table.shape[-3]
|
| 106 |
+
a, b = hidden_states[..., :pairs], hidden_states[..., pairs:2 * pairs]
|
| 107 |
+
t = rotation_table
|
| 108 |
+
first = t[..., 0, 0] * a + t[..., 0, 1] * b
|
| 109 |
+
second = t[..., 1, 0] * a + t[..., 1, 1] * b
|
| 110 |
+
return torch.cat((first, second, hidden_states[..., 2 * pairs:]), dim=-1)
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
def speed_patches() -> None:
|
| 114 |
+
"""Inference speed-ups applied without editing the vendored musubi-tuner.
|
| 115 |
+
|
| 116 |
+
- The int8 matmul kernels autotune with ``key=["m", "n", "k"]``; ``m`` is the packed row count, so
|
| 117 |
+
every new prompt length, resolution or two-track region size re-ran the 6-config benchmark
|
| 118 |
+
(about 4 s; a first step of 6.2 s against 2.08 s). Keyed on ``n, k`` only, each layer shape tunes once.
|
| 119 |
+
- The block RoPE uses the element-wise ``rope_split_half``.
|
| 120 |
+
"""
|
| 121 |
+
from musubi_tuner.minimax_h3 import model as h3_model
|
| 122 |
+
from musubi_tuner.modules import convrot_int8_kernels as kernels
|
| 123 |
+
|
| 124 |
+
for name in ("_int8_matmul_dequant_kernel", "_int8_matmul_dequant_per_row_kernel"):
|
| 125 |
+
kernel = getattr(kernels, name, None)
|
| 126 |
+
if kernel is not None and "m" in getattr(kernel, "keys", ()):
|
| 127 |
+
kernel.keys = [key for key in kernel.keys if key != "m"]
|
| 128 |
+
h3_model._apply_rope_split_half = rope_split_half
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
def load_dit(checkpoint=LOT_H3_CHECKPOINT, *, device="cuda", swap: int = 0, train: bool = False,
|
| 132 |
+
pinned: bool = True):
|
| 133 |
+
"""The H3 transformer, frozen, ready for ``predict``.
|
| 134 |
+
|
| 135 |
+
``swap`` blocks stream from CPU (H2D only: the base is frozen). ``train``
|
| 136 |
+
turns on activation checkpointing and lets the swap run a backward pass.
|
| 137 |
+
``pinned`` keeps the streamed blocks in pinned host memory (about 0.42 GB each), so each block is one
|
| 138 |
+
async copy; unpinned, a worker thread first copies it into a staging buffer (about 4.6 GB/s), which
|
| 139 |
+
made a 1,764-token step wait on transfers (0.73 s). ``LOT_H3_PINNED=0`` turns it off.
|
| 140 |
+
"""
|
| 141 |
+
speed_patches()
|
| 142 |
+
pinned = pinned and os.environ.get("LOT_H3_PINNED", "1") != "0"
|
| 143 |
+
from musubi_tuner.minimax_h3.model import load_h3_transformer
|
| 144 |
+
from musubi_tuner.modules.custom_offloading_utils import BlockSwapConfig
|
| 145 |
+
|
| 146 |
+
device = torch.device(device)
|
| 147 |
+
dit = load_h3_transformer(str(checkpoint), device="cpu" if swap else device)
|
| 148 |
+
dit.requires_grad_(False)
|
| 149 |
+
if swap:
|
| 150 |
+
dit.enable_block_swap(swap, BlockSwapConfig(device=device, supports_backward=train, h2d_only=True,
|
| 151 |
+
use_pinned_memory=pinned))
|
| 152 |
+
dit.move_to_device_except_swap_blocks(device)
|
| 153 |
+
dit.prepare_block_swap_before_forward()
|
| 154 |
+
if train:
|
| 155 |
+
dit.enable_gradient_checkpointing()
|
| 156 |
+
dit.train()
|
| 157 |
+
return dit
|
| 158 |
+
|
| 159 |
+
|
| 160 |
+
def make_lora(dit, rank: int):
|
| 161 |
+
"""A rank-``rank`` LoRA (alpha = rank) on qkv, out, fc1, fc2 of every block, applied to ``dit``."""
|
| 162 |
+
from musubi_tuner.networks import lora_minimax_h3
|
| 163 |
+
|
| 164 |
+
network = lora_minimax_h3.create_arch_network(1.0, rank, float(rank), None, [], dit)
|
| 165 |
+
network.apply_to(None, dit, apply_text_encoder=False, apply_unet=True)
|
| 166 |
+
expected = 4 * len(dit.blocks)
|
| 167 |
+
if len(network.unet_loras) != expected:
|
| 168 |
+
raise RuntimeError(f"{len(network.unet_loras)} LoRA modules wrapped, expected {expected}")
|
| 169 |
+
return network
|
| 170 |
+
|
| 171 |
+
|
| 172 |
+
def set_lora(network, on: bool) -> None:
|
| 173 |
+
for module in network.unet_loras:
|
| 174 |
+
module.multiplier = 1.0 if on else 0.0
|
| 175 |
+
|
| 176 |
+
|
| 177 |
+
# ------------------------------------------------------------------------------ forward
|
| 178 |
+
|
| 179 |
+
def _sync_swap(dit) -> None:
|
| 180 |
+
"""Put a streaming offloader in forward-only mode under no_grad and back for a training forward."""
|
| 181 |
+
if not dit.blocks_to_swap or not dit.offloader.supports_backward:
|
| 182 |
+
return
|
| 183 |
+
forward_only = not torch.is_grad_enabled()
|
| 184 |
+
if dit.offloader.forward_only != forward_only:
|
| 185 |
+
if forward_only:
|
| 186 |
+
dit.switch_block_swap_for_inference()
|
| 187 |
+
else:
|
| 188 |
+
dit.switch_block_swap_for_training()
|
| 189 |
+
|
| 190 |
+
|
| 191 |
+
def _layout(text_length: int, latent_t: int, lat_h: int, lat_w: int):
|
| 192 |
+
from musubi_tuner.minimax_h3.packing import build_h3_layout
|
| 193 |
+
|
| 194 |
+
return build_h3_layout(task="t2va", text_length=text_length, target_video=(latent_t, lat_h, lat_w),
|
| 195 |
+
target_audio_frames=audio_frames(latent_t), one_frame=latent_t == 1)
|
| 196 |
+
|
| 197 |
+
|
| 198 |
+
def _silence(latent_t: int, sigma_a: float, device, generator=None) -> torch.Tensor:
|
| 199 |
+
"""``(1, 32, 2, A)`` silent audio noised to ``sigma_a``."""
|
| 200 |
+
eps = torch.randn(1, AUDIO_CHANNELS, 2, audio_frames(latent_t), device=device, generator=generator)
|
| 201 |
+
return sigma_a * eps
|
| 202 |
+
|
| 203 |
+
|
| 204 |
+
def predict(dit, latent: torch.Tensor, t, text: torch.Tensor, *, lot=None,
|
| 205 |
+
audio_generator: torch.Generator | None = None, dual=None, capture: dict | None = None) -> torch.Tensor:
|
| 206 |
+
"""H3's velocity (``x0 - eps``) for ``latent`` ``(1, 24, T, H, W)`` at cleanness ``t``.
|
| 207 |
+
|
| 208 |
+
``text`` is ``(1, L, 5120)`` (or ``(L, 5120)``) Qwen3-VL layer-50 states.
|
| 209 |
+
With ``lot`` (a ``LotSplice``), ``latent`` is the splice's ``y_t`` and the
|
| 210 |
+
target video runs on its layout.
|
| 211 |
+
"""
|
| 212 |
+
if latent.ndim != 5 or latent.shape[0] != 1:
|
| 213 |
+
raise ValueError(f"latent must be (1, 24, T, H, W), got {tuple(latent.shape)}")
|
| 214 |
+
if text.ndim == 2:
|
| 215 |
+
text = text.unsqueeze(0)
|
| 216 |
+
_sync_swap(dit)
|
| 217 |
+
t_video = float(torch.as_tensor(t).reshape(-1)[0])
|
| 218 |
+
sigma_a = float(audio_sigma(1.0 - t_video))
|
| 219 |
+
_b, _c, latent_t, lat_h, lat_w = latent.shape
|
| 220 |
+
layout = _layout(text.shape[1], latent_t, lat_h, lat_w)
|
| 221 |
+
device = dit.video_patch_proj.weight.device
|
| 222 |
+
audio = _silence(latent_t, sigma_a, device, audio_generator)
|
| 223 |
+
tags = torch.ones(1, text.shape[1], dtype=torch.int64)
|
| 224 |
+
if lot is None:
|
| 225 |
+
if dual is not None or capture is not None:
|
| 226 |
+
raise ValueError("dual and capture need a LoT splice (use a dense layout for the dense model)")
|
| 227 |
+
return dit(video_latents=latent, audio_latents=audio, text_hidden_states=text, text_token_tags=tags,
|
| 228 |
+
layout=layout, model_t_video=t_video, model_t_audio=1.0 - sigma_a).video
|
| 229 |
+
return _lot_forward(dit, lot, layout, latent, audio, text, tags, t_video, 1.0 - sigma_a, dual=dual, capture=capture)
|
| 230 |
+
|
| 231 |
+
|
| 232 |
+
def _lot_forward(dit, lot, layout, latent, audio, text, tags, t_video: float, t_audio: float, *,
|
| 233 |
+
dual=None, capture: dict | None = None) -> torch.Tensor:
|
| 234 |
+
"""``MiniMaxH3Model.forward`` for a T2VA layout, with the target video on ``lot.layout``."""
|
| 235 |
+
from musubi_tuner.minimax_h3.packing import (
|
| 236 |
+
H3RowSegment,
|
| 237 |
+
build_position_grid,
|
| 238 |
+
build_timestep_rows,
|
| 239 |
+
pack_audio_rows,
|
| 240 |
+
pack_video_rows,
|
| 241 |
+
unpack_targets,
|
| 242 |
+
)
|
| 243 |
+
from musubi_tuner.utils.model_utils import create_cpu_offloading_wrapper
|
| 244 |
+
|
| 245 |
+
if layout.task != "t2va":
|
| 246 |
+
raise ValueError("the LoT tail is wired for T2VA layouts only")
|
| 247 |
+
video_segment = layout.target_video_segment
|
| 248 |
+
if video_segment.stop != layout.row_count:
|
| 249 |
+
raise RuntimeError("the target video must be the last segment of the packed sequence")
|
| 250 |
+
device = dit.video_patch_proj.weight.device
|
| 251 |
+
latent_t, lat_h, lat_w = latent.shape[2:]
|
| 252 |
+
|
| 253 |
+
# 1. Target video: extent heads instead of video_patch_proj.
|
| 254 |
+
tokens = lot.video_tokens(pack_video_rows(latent.to(device, torch.float32))[0], latent_t, lat_h, lat_w)
|
| 255 |
+
video = lot.embed_rows(tokens).to(dit.dtype).unsqueeze(0)
|
| 256 |
+
count = video.shape[1]
|
| 257 |
+
segments = layout.segments[:-1] + (H3RowSegment(video_segment.role, video_segment.kind,
|
| 258 |
+
video_segment.start, video_segment.start + count),)
|
| 259 |
+
lot_layout = replace(layout, segments=segments, row_count=video_segment.start + count)
|
| 260 |
+
|
| 261 |
+
audio_rows = dit.audio_patch_proj(pack_audio_rows(audio.to(device)).to(torch.float32)).to(dit.dtype)
|
| 262 |
+
text_rows = dit.token_refiner(dit.condition_proj(text.to(device, dit.dtype)))
|
| 263 |
+
hidden = torch.cat([text_rows, audio_rows, video], dim=1)
|
| 264 |
+
|
| 265 |
+
# 2. Rotary positions: the dense grid with the video tail replaced by rectangle centres.
|
| 266 |
+
dense_positions = build_position_grid(layout)[0]
|
| 267 |
+
positions = lot.replace_video_positions(dense_positions, latent_t, lat_h, lat_w)
|
| 268 |
+
rotation = dit._rotation_table(positions.unsqueeze(0).to(device), hidden.dtype)
|
| 269 |
+
patch = contextlib.nullcontext()
|
| 270 |
+
if dual is not None or (capture is not None and "kv" in capture):
|
| 271 |
+
from mixed_res import DualKVPlan, patched_attention
|
| 272 |
+
|
| 273 |
+
n_dense = latent_t * (lat_h // 2) * (lat_w // 2)
|
| 274 |
+
dense_rotation = dit._rotation_table(dense_positions[-n_dense:].unsqueeze(0).to(device), hidden.dtype)
|
| 275 |
+
plan = dual.plan(lot.layout, device) if dual is not None else DualKVPlan(lot.layout).to(device)
|
| 276 |
+
patch = patched_attention(dit.blocks, plan=plan, maps=None if dual is None else dual.maps,
|
| 277 |
+
video_start=video_segment.start, dense_rotation=dense_rotation,
|
| 278 |
+
capture=None if capture is None else capture.get("kv"))
|
| 279 |
+
rows = build_timestep_rows(lot_layout, text_token_tags=tags, model_t_video=t_video, model_t_audio=t_audio)
|
| 280 |
+
temb = dit._timestep_embeddings(rows.unique_timesteps, device)
|
| 281 |
+
|
| 282 |
+
with patch:
|
| 283 |
+
for index, block in enumerate(dit.blocks):
|
| 284 |
+
if dit.blocks_to_swap:
|
| 285 |
+
dit.offloader.wait_for_block(index)
|
| 286 |
+
if dit.gradient_checkpointing and dit.training:
|
| 287 |
+
forward = block
|
| 288 |
+
if dit.activation_cpu_offloading:
|
| 289 |
+
forward = create_cpu_offloading_wrapper(forward, device)
|
| 290 |
+
hidden = torch.utils.checkpoint.checkpoint(forward, hidden, temb, rows.block_segments, rotation,
|
| 291 |
+
use_reentrant=False)
|
| 292 |
+
else:
|
| 293 |
+
hidden = block(hidden, temb, rows.block_segments, rotation)
|
| 294 |
+
if dit.blocks_to_swap:
|
| 295 |
+
dit.offloader.submit_move_blocks_forward(dit.blocks, index)
|
| 296 |
+
if capture is not None and capture.get("block") == index:
|
| 297 |
+
capture["states"] = hidden[0, video_segment.start:].detach().float()
|
| 298 |
+
hidden = hidden.to(device)
|
| 299 |
+
|
| 300 |
+
# 3. Final layer: shared norm + AdaLN, then the extent heads in place of video_out.
|
| 301 |
+
final = dit.final_layer
|
| 302 |
+
shift, scale = final.adaln_proj(temb)
|
| 303 |
+
audio_index = rows.audio_timestep_index
|
| 304 |
+
audio_states = final.norm(hidden[:, layout.target_audio_segment.row_slice])
|
| 305 |
+
audio_states = audio_states * (1.0 + scale[audio_index]) + shift[audio_index]
|
| 306 |
+
audio_out = final.audio_out(audio_states.to(torch.float32))
|
| 307 |
+
modulated = lot.modulate(final, hidden[0, lot_layout.target_video_segment.row_slice], temb,
|
| 308 |
+
rows.video_timestep_index)
|
| 309 |
+
velocity = lot.project_rows(modulated, tokens, max(0.0, min(1.0, 1.0 - t_video)))
|
| 310 |
+
video_out, _audio = unpack_targets(layout, velocity.unsqueeze(0).to(torch.float32), audio_out)
|
| 311 |
+
return video_out.to(latent.dtype)
|
| 312 |
+
|
| 313 |
+
|
| 314 |
+
# ------------------------------------------------------------------------------ sampling
|
| 315 |
+
|
| 316 |
+
def sigma_schedule(steps: int, shift: float = VIDEO_SHIFT) -> torch.Tensor:
|
| 317 |
+
"""``steps + 1`` sigmas from 1 to 0, linear in the base schedule, then shifted."""
|
| 318 |
+
return shift_sigma(torch.linspace(1.0, 0.0, steps + 1, dtype=torch.float64), shift)
|
| 319 |
+
|
| 320 |
+
|
| 321 |
+
@torch.no_grad()
|
| 322 |
+
def sample(dit, text: torch.Tensor, *, height: int, width: int, latent_t: int = 1, steps: int = 20,
|
| 323 |
+
seed: int = 0, lot=None, device="cuda", on_step=None) -> torch.Tensor:
|
| 324 |
+
"""Euler sampling (``eta = 0``) of a ``(1, 24, T, height/16, width/16)`` latent.
|
| 325 |
+
|
| 326 |
+
Same update as diffusers' ``MiniMaxH3Scheduler``: ``x0 = x + sigma * v``,
|
| 327 |
+
then ``x <- r * x + (1 - r) * x0`` with ``r = sigma_next / sigma``. With
|
| 328 |
+
``lot`` the state is y-space; map the result back with ``lot.to_x``.
|
| 329 |
+
"""
|
| 330 |
+
generator = torch.Generator(device).manual_seed(seed)
|
| 331 |
+
x = torch.randn(1, 24, latent_t, height // 16, width // 16, device=device, generator=generator)
|
| 332 |
+
audio_generator = torch.Generator(device).manual_seed(seed + 1)
|
| 333 |
+
sigmas = sigma_schedule(steps)
|
| 334 |
+
for index in range(steps):
|
| 335 |
+
sigma, sigma_next = float(sigmas[index]), float(sigmas[index + 1])
|
| 336 |
+
velocity = predict(dit, x, 1.0 - sigma, text, lot=lot, audio_generator=audio_generator).float()
|
| 337 |
+
x0 = x + sigma * velocity
|
| 338 |
+
ratio = sigma_next / sigma
|
| 339 |
+
x = ratio * x + (1.0 - ratio) * x0
|
| 340 |
+
if on_step is not None:
|
| 341 |
+
on_step(index, sigma)
|
| 342 |
+
return x
|
| 343 |
+
|
| 344 |
+
|
| 345 |
+
# ---------------------------------------------------------------------------------- VAE
|
| 346 |
+
|
| 347 |
+
def load_vae(path, device="cuda", *, decoder: bool = True, dtype: torch.dtype = torch.float16):
|
| 348 |
+
"""H3's video VAE.
|
| 349 |
+
|
| 350 |
+
``encode`` takes ``(1, 3, [F,] H, W)`` pixels in [-1, 1] and returns normalized
|
| 351 |
+
latents; ``decode`` takes normalized latents and returns ``(1, 3, F, H, W)``
|
| 352 |
+
pixels in [-1, 1]. ``decoder=False`` skips the decoder weights (encode only).
|
| 353 |
+
"""
|
| 354 |
+
from musubi_tuner.minimax_h3.video_vae import load_video_vae
|
| 355 |
+
|
| 356 |
+
return load_video_vae(str(path), device=device, dtype=dtype, load_decoder=decoder)
|
lot/h3_positions.py
ADDED
|
@@ -0,0 +1,168 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Rotary positions for an H3 target-video tail.
|
| 2 |
+
|
| 3 |
+
H3 does not feed integer token indices to RoPE. Each uniform token sits on a
|
| 4 |
+
spatial axis scaled by 32 and a non-uniform time axis whose steps are
|
| 5 |
+
``5/3 * (1, 4, 4, 4, 4)``. A LoT token is placed at its center on that same
|
| 6 |
+
grid. A 1x1 token lands on the pretrained coordinate.
|
| 7 |
+
|
| 8 |
+
The packed order here is text, then stereo audio, then target video. There
|
| 9 |
+
are no reference frames and no keyframe rows. This module does not import
|
| 10 |
+
the Separable Causal Diffusion port.
|
| 11 |
+
"""
|
| 12 |
+
|
| 13 |
+
from __future__ import annotations
|
| 14 |
+
|
| 15 |
+
import math
|
| 16 |
+
|
| 17 |
+
import torch
|
| 18 |
+
|
| 19 |
+
from layout import LotLayout
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
FRAME_PER_TOKEN = (1, 4, 4, 4, 4)
|
| 23 |
+
FRAME_RESCALE = 5.0 / 3.0
|
| 24 |
+
ROPE_SPATIAL_SCALE = 32.0
|
| 25 |
+
AUDIO_CHANNELS = 2
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def spatial_axis(dim: int, patch: int, sqrt_area: float) -> torch.Tensor:
|
| 29 |
+
"""``dim // patch`` coordinates. A square canvas spans ``[0, 32)``."""
|
| 30 |
+
if patch < 1 or dim % patch != 0:
|
| 31 |
+
raise ValueError(f"dim {dim} is not divisible by patch {patch}")
|
| 32 |
+
count = dim // patch
|
| 33 |
+
ratio = float(dim) / float(sqrt_area)
|
| 34 |
+
index = torch.arange(count, dtype=torch.float64)
|
| 35 |
+
return (index * (ratio / count) + (1.0 - ratio) / 2.0) * ROPE_SPATIAL_SCALE
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def sample_axis(axis: torch.Tensor, coord: float) -> torch.Tensor:
|
| 39 |
+
"""Linear sample. Integer coordinates return the stored sample."""
|
| 40 |
+
last = axis.shape[0] - 1
|
| 41 |
+
if last < 0:
|
| 42 |
+
raise ValueError("empty rotary axis")
|
| 43 |
+
if coord < -1e-6 or coord > last + 1e-6:
|
| 44 |
+
raise ValueError(f"coordinate {coord} is outside 0..{last}")
|
| 45 |
+
left = int(math.floor(coord))
|
| 46 |
+
if left >= last:
|
| 47 |
+
return axis[last]
|
| 48 |
+
frac = coord - left
|
| 49 |
+
return axis[left] * (1.0 - frac) + axis[left + 1] * frac
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def video_time(num_frames: int, origin: float) -> torch.Tensor:
|
| 53 |
+
"""Rotary time of every latent frame, starting at ``origin``."""
|
| 54 |
+
if num_frames < 1:
|
| 55 |
+
raise ValueError("num_frames must be positive")
|
| 56 |
+
spans = torch.tensor(
|
| 57 |
+
[FRAME_RESCALE * FRAME_PER_TOKEN[index % 5] for index in range(num_frames)],
|
| 58 |
+
dtype=torch.float64,
|
| 59 |
+
)
|
| 60 |
+
return float(origin) + torch.cat([torch.zeros(1, dtype=torch.float64), spans[:-1].cumsum(0)])
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def video_positions(layout: LotLayout, latent_height: int, latent_width: int, origin: float) -> torch.Tensor:
|
| 64 |
+
"""``(L, 3)`` positions in ``(t, h, w)`` for ``layout.rects`` order.
|
| 65 |
+
|
| 66 |
+
``latent_height`` and ``latent_width`` are VAE-latent pixels. The uniform
|
| 67 |
+
token grid is those sizes divided by the native spatial patch of 2.
|
| 68 |
+
"""
|
| 69 |
+
token_h = latent_height // 2
|
| 70 |
+
token_w = latent_width // 2
|
| 71 |
+
if latent_height % 2 or latent_width % 2:
|
| 72 |
+
raise ValueError("latent sides must be divisible by the 2x2 patch")
|
| 73 |
+
if layout.height != token_h or layout.width != token_w:
|
| 74 |
+
raise ValueError(
|
| 75 |
+
f"layout {(layout.height, layout.width)} does not match latent "
|
| 76 |
+
f"{(latent_height, latent_width)}"
|
| 77 |
+
)
|
| 78 |
+
sqrt_area = math.sqrt(latent_height * latent_width)
|
| 79 |
+
height_axis = spatial_axis(latent_height, 2, sqrt_area)
|
| 80 |
+
width_axis = spatial_axis(latent_width, 2, sqrt_area)
|
| 81 |
+
times = video_time(layout.time, origin)
|
| 82 |
+
rows = []
|
| 83 |
+
for rect in layout.rects:
|
| 84 |
+
if rect.et != 1:
|
| 85 |
+
raise ValueError("this H3 grid keeps a temporal extent of 1")
|
| 86 |
+
rows.append((
|
| 87 |
+
times[rect.t],
|
| 88 |
+
sample_axis(height_axis, rect.u + (rect.eh - 1) / 2.0),
|
| 89 |
+
sample_axis(width_axis, rect.v + (rect.ew - 1) / 2.0),
|
| 90 |
+
))
|
| 91 |
+
return torch.stack([torch.stack(row) for row in rows], dim=0)
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
def _frame_rows(latent_height: int, latent_width: int) -> torch.Tensor:
|
| 95 |
+
sqrt_area = math.sqrt(latent_height * latent_width)
|
| 96 |
+
height = spatial_axis(latent_height, 2, sqrt_area)
|
| 97 |
+
width = spatial_axis(latent_width, 2, sqrt_area)
|
| 98 |
+
hh, ww = torch.meshgrid(height, width, indexing="ij")
|
| 99 |
+
return torch.stack([hh.reshape(-1), ww.reshape(-1)], dim=-1)
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
def _span_sum(num_frames: int) -> float:
|
| 103 |
+
return float(sum(FRAME_RESCALE * FRAME_PER_TOKEN[index % 5] for index in range(num_frames)))
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
def packed_positions(
|
| 107 |
+
layout: LotLayout,
|
| 108 |
+
latent_height: int,
|
| 109 |
+
latent_width: int,
|
| 110 |
+
text_len: int,
|
| 111 |
+
num_audio_latents: int,
|
| 112 |
+
keyframes: list[int] | None = None,
|
| 113 |
+
refs: list[tuple] | None = None,
|
| 114 |
+
) -> tuple[torch.Tensor, int]:
|
| 115 |
+
"""``(S, 3)`` positions and the index where target video begins.
|
| 116 |
+
|
| 117 |
+
Order matches H3's packing (and musubi-tuner's ``build_position_grid``): text, keyframes, refs, audio, target video. Audio
|
| 118 |
+
does not advance the video clock. With no keyframes and no refs both start
|
| 119 |
+
at ``text_len``. A reference image advances the cursor by 1. A video-kind
|
| 120 |
+
reference ``(h, w, t)`` with ``t > 1`` advances it by the video span sum.
|
| 121 |
+
Keyframe rows are inserted before the refs, but their time is the cursor
|
| 122 |
+
after those refs, plus ``5/3 * frame_index``.
|
| 123 |
+
"""
|
| 124 |
+
if text_len < 0 or num_audio_latents < 0:
|
| 125 |
+
raise ValueError("text and audio lengths must be non-negative")
|
| 126 |
+
cursor = float(text_len)
|
| 127 |
+
text = torch.zeros(text_len, 3, dtype=torch.float64)
|
| 128 |
+
if text_len:
|
| 129 |
+
text[:, 0] = torch.arange(text_len, dtype=torch.float64)
|
| 130 |
+
target_frame = _frame_rows(latent_height, latent_width)
|
| 131 |
+
ref_rows = []
|
| 132 |
+
for ref in refs or ():
|
| 133 |
+
rh, rw = int(ref[0]), int(ref[1])
|
| 134 |
+
rt = int(ref[2]) if len(ref) > 2 else 1
|
| 135 |
+
frame = _frame_rows(rh, rw)
|
| 136 |
+
if rt > 1:
|
| 137 |
+
times = video_time(rt, cursor)
|
| 138 |
+
block = torch.empty(rt * frame.shape[0], 3, dtype=torch.float64)
|
| 139 |
+
block[:, 0] = times.repeat_interleave(frame.shape[0])
|
| 140 |
+
block[:, 1:] = frame.repeat(rt, 1)
|
| 141 |
+
ref_rows.append(block)
|
| 142 |
+
cursor += _span_sum(rt)
|
| 143 |
+
continue
|
| 144 |
+
block = torch.empty(frame.shape[0], 3, dtype=torch.float64)
|
| 145 |
+
block[:, 0] = cursor
|
| 146 |
+
block[:, 1:] = frame
|
| 147 |
+
ref_rows.append(block)
|
| 148 |
+
cursor += 1.0
|
| 149 |
+
parts = [text]
|
| 150 |
+
for index in keyframes or ():
|
| 151 |
+
block = torch.empty(target_frame.shape[0], 3, dtype=torch.float64)
|
| 152 |
+
block[:, 0] = cursor + FRAME_RESCALE * float(index)
|
| 153 |
+
block[:, 1:] = target_frame
|
| 154 |
+
parts.append(block)
|
| 155 |
+
parts.extend(ref_rows)
|
| 156 |
+
if num_audio_latents:
|
| 157 |
+
sqrt_area = math.sqrt(latent_height * latent_width)
|
| 158 |
+
width_axis = spatial_axis(latent_width, 2, sqrt_area)
|
| 159 |
+
audio = torch.zeros(num_audio_latents * AUDIO_CHANNELS, 3, dtype=torch.float64)
|
| 160 |
+
audio[:, 0] = (cursor + torch.arange(num_audio_latents, dtype=torch.float64)).repeat(AUDIO_CHANNELS)
|
| 161 |
+
audio[:, 2] = torch.cat([
|
| 162 |
+
torch.full((num_audio_latents,), float(width_axis[0]), dtype=torch.float64),
|
| 163 |
+
torch.full((num_audio_latents,), float(width_axis[-1]), dtype=torch.float64),
|
| 164 |
+
])
|
| 165 |
+
parts.append(audio)
|
| 166 |
+
video_start = sum(part.shape[0] for part in parts)
|
| 167 |
+
parts.append(video_positions(layout, latent_height, latent_width, cursor))
|
| 168 |
+
return torch.cat(parts, dim=0), video_start
|
lot/layout.py
ADDED
|
@@ -0,0 +1,385 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Level-of-Token layouts.
|
| 2 |
+
|
| 3 |
+
A layout is a partition of the uniform token lattice into axis-aligned
|
| 4 |
+
rectangles. Extents are measured in uniform tokens, matching Kiyohiro Nakayama, Brian Chao, Jan Ackermann, Hansheng Chen, Federico Tombari, Leonidas Guibas, Lior Yariv and Gordon Wetzstein.
|
| 5 |
+
2026, Sec. 3.1. Video layouts in the paper keep a temporal extent of 1 and
|
| 6 |
+
vary only the spatial rectangle (Appendix A.10).
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
import math
|
| 12 |
+
from dataclasses import dataclass
|
| 13 |
+
|
| 14 |
+
import torch
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
@dataclass(frozen=True)
|
| 18 |
+
class TokenRect:
|
| 19 |
+
"""One LoT token covering ``[t, t+et) x [u, u+eh) x [v, v+ew)``."""
|
| 20 |
+
|
| 21 |
+
t: int
|
| 22 |
+
u: int
|
| 23 |
+
v: int
|
| 24 |
+
et: int
|
| 25 |
+
eh: int
|
| 26 |
+
ew: int
|
| 27 |
+
|
| 28 |
+
@property
|
| 29 |
+
def extent(self) -> tuple[int, int, int]:
|
| 30 |
+
return (self.et, self.eh, self.ew)
|
| 31 |
+
|
| 32 |
+
def center(self) -> tuple[float, float, float]:
|
| 33 |
+
"""Finest-grid center. Unit extents recover the integer lattice (eq. 13)."""
|
| 34 |
+
return (
|
| 35 |
+
self.t + (self.et - 1) / 2,
|
| 36 |
+
self.u + (self.eh - 1) / 2,
|
| 37 |
+
self.v + (self.ew - 1) / 2,
|
| 38 |
+
)
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
@dataclass(frozen=True)
|
| 42 |
+
class LotLayout:
|
| 43 |
+
time: int
|
| 44 |
+
height: int
|
| 45 |
+
width: int
|
| 46 |
+
rects: tuple[TokenRect, ...]
|
| 47 |
+
|
| 48 |
+
def __post_init__(self) -> None:
|
| 49 |
+
validate_layout(self)
|
| 50 |
+
|
| 51 |
+
@property
|
| 52 |
+
def count(self) -> int:
|
| 53 |
+
return len(self.rects)
|
| 54 |
+
|
| 55 |
+
@property
|
| 56 |
+
def dense_count(self) -> int:
|
| 57 |
+
return self.time * self.height * self.width
|
| 58 |
+
|
| 59 |
+
@property
|
| 60 |
+
def compression(self) -> float:
|
| 61 |
+
"""Uniform-token count divided by the LoT sequence length."""
|
| 62 |
+
return self.dense_count / self.count
|
| 63 |
+
|
| 64 |
+
def centers(self) -> torch.Tensor:
|
| 65 |
+
"""``(L, 3)`` centers in ``(t, h, w)`` order on the uniform lattice."""
|
| 66 |
+
return torch.tensor([rect.center() for rect in self.rects], dtype=torch.float32)
|
| 67 |
+
|
| 68 |
+
def groups(self) -> list[tuple[tuple[int, int, int], list[int]]]:
|
| 69 |
+
"""Rect indices grouped by extent, in order of first appearance."""
|
| 70 |
+
order: list[tuple[int, int, int]] = []
|
| 71 |
+
buckets: dict[tuple[int, int, int], list[int]] = {}
|
| 72 |
+
for index, rect in enumerate(self.rects):
|
| 73 |
+
bucket = buckets.get(rect.extent)
|
| 74 |
+
if bucket is None:
|
| 75 |
+
order.append(rect.extent)
|
| 76 |
+
bucket = []
|
| 77 |
+
buckets[rect.extent] = bucket
|
| 78 |
+
bucket.append(index)
|
| 79 |
+
return [(extent, buckets[extent]) for extent in order]
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
def validate_layout(layout: LotLayout) -> None:
|
| 83 |
+
if layout.time < 1 or layout.height < 1 or layout.width < 1:
|
| 84 |
+
raise ValueError("layout axes must be positive")
|
| 85 |
+
if not layout.rects:
|
| 86 |
+
raise ValueError("layout has no tokens")
|
| 87 |
+
cover = torch.zeros(layout.time, layout.height, layout.width, dtype=torch.int32)
|
| 88 |
+
for rect in layout.rects:
|
| 89 |
+
if min(rect.et, rect.eh, rect.ew) < 1:
|
| 90 |
+
raise ValueError(f"non-positive extent on {rect}")
|
| 91 |
+
t1 = rect.t + rect.et
|
| 92 |
+
u1 = rect.u + rect.eh
|
| 93 |
+
v1 = rect.v + rect.ew
|
| 94 |
+
if rect.t < 0 or rect.u < 0 or rect.v < 0 or t1 > layout.time or u1 > layout.height or v1 > layout.width:
|
| 95 |
+
raise ValueError(f"rect {rect} leaves the {layout.time}x{layout.height}x{layout.width} lattice")
|
| 96 |
+
cover[rect.t:t1, rect.u:u1, rect.v:v1] += 1
|
| 97 |
+
if int(cover.min()) != 1 or int(cover.max()) != 1:
|
| 98 |
+
raise ValueError("rects are not a partition of the uniform token lattice")
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
def dense_layout(time: int, height: int, width: int) -> LotLayout:
|
| 102 |
+
"""One 1x1x1 token per uniform site, in t-then-row-major order."""
|
| 103 |
+
rects = [
|
| 104 |
+
TokenRect(t, u, v, 1, 1, 1)
|
| 105 |
+
for t in range(time)
|
| 106 |
+
for u in range(height)
|
| 107 |
+
for v in range(width)
|
| 108 |
+
]
|
| 109 |
+
return LotLayout(time, height, width, tuple(rects))
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
def layout_from_rects(time: int, height: int, width: int, rects: list[TokenRect]) -> LotLayout:
|
| 113 |
+
return LotLayout(time, height, width, tuple(rects))
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
def _walk_squares(time: int, height: int, width: int, root: int, keep, edge_rects: bool = False) -> LotLayout:
|
| 117 |
+
"""Quadtree over any ``height x width`` grid; ``keep(t, u, v, eh, ew)`` decides a merge.
|
| 118 |
+
|
| 119 |
+
Blocks are laid on the ``root`` lattice from the top-left. A block that
|
| 120 |
+
crosses the far edge is clipped: it splits until its pieces fit (squares
|
| 121 |
+
only), or, with ``edge_rects``, is kept as the clipped rectangle when both
|
| 122 |
+
sides are powers of two (e.g. 4x2 on the 2-token strip of H3's 24x42).
|
| 123 |
+
"""
|
| 124 |
+
if root < 1 or (root & (root - 1)) != 0:
|
| 125 |
+
raise ValueError(f"root extent must be a power of two, got {root}")
|
| 126 |
+
rects: list[TokenRect] = []
|
| 127 |
+
|
| 128 |
+
def visit(t: int, u: int, v: int, size: int) -> None:
|
| 129 |
+
if u >= height or v >= width:
|
| 130 |
+
return
|
| 131 |
+
eh, ew = min(size, height - u), min(size, width - v)
|
| 132 |
+
if size == 1:
|
| 133 |
+
rects.append(TokenRect(t, u, v, 1, 1, 1))
|
| 134 |
+
return
|
| 135 |
+
if eh == size and ew == size:
|
| 136 |
+
if keep(t, u, v, size, size):
|
| 137 |
+
rects.append(TokenRect(t, u, v, 1, size, size))
|
| 138 |
+
return
|
| 139 |
+
elif edge_rects and max(eh, ew) > 1 and _power_of_two(eh) and _power_of_two(ew) and keep(t, u, v, eh, ew):
|
| 140 |
+
rects.append(TokenRect(t, u, v, 1, eh, ew))
|
| 141 |
+
return
|
| 142 |
+
half = size // 2
|
| 143 |
+
for du in (0, half):
|
| 144 |
+
for dv in (0, half):
|
| 145 |
+
visit(t, u + du, v + dv, half)
|
| 146 |
+
|
| 147 |
+
for t in range(time):
|
| 148 |
+
for u in range(0, height, root):
|
| 149 |
+
for v in range(0, width, root):
|
| 150 |
+
visit(t, u, v, root)
|
| 151 |
+
return LotLayout(time, height, width, tuple(rects))
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
def _power_of_two(n: int) -> bool:
|
| 155 |
+
return n >= 1 and (n & (n - 1)) == 0
|
| 156 |
+
|
| 157 |
+
|
| 158 |
+
def _as_time_field(field: torch.Tensor) -> torch.Tensor:
|
| 159 |
+
if field.ndim == 2:
|
| 160 |
+
return field.unsqueeze(0)
|
| 161 |
+
if field.ndim == 3:
|
| 162 |
+
return field
|
| 163 |
+
raise ValueError(f"expected (H, W) or (T, H, W), got {tuple(field.shape)}")
|
| 164 |
+
|
| 165 |
+
|
| 166 |
+
def layout_from_detail(
|
| 167 |
+
detail: torch.Tensor,
|
| 168 |
+
thresholds: dict[int, float],
|
| 169 |
+
root: int = 8,
|
| 170 |
+
edge_rects: bool = False,
|
| 171 |
+
) -> LotLayout:
|
| 172 |
+
"""Quadtree of eq. 23.
|
| 173 |
+
|
| 174 |
+
A block of extent ``b`` splits when its maximum detail is at least
|
| 175 |
+
``thresholds[b]``. Thresholds must be nondecreasing as blocks get finer:
|
| 176 |
+
``t_root <= ... <= t_2``. ``detail`` is already on the uniform token lattice.
|
| 177 |
+
"""
|
| 178 |
+
field = _as_time_field(detail).detach().float()
|
| 179 |
+
sizes = [size for size in (8, 4, 2) if size <= root]
|
| 180 |
+
missing = [size for size in sizes if size not in thresholds]
|
| 181 |
+
if missing:
|
| 182 |
+
raise ValueError(f"thresholds missing extents {missing}")
|
| 183 |
+
ordered = [float(thresholds[size]) for size in sizes]
|
| 184 |
+
if ordered != sorted(ordered):
|
| 185 |
+
raise ValueError("thresholds must satisfy t8 <= t4 <= t2")
|
| 186 |
+
|
| 187 |
+
def keep(t: int, u: int, v: int, eh: int, ew: int) -> bool:
|
| 188 |
+
block = field[t, u:u + eh, v:v + ew]
|
| 189 |
+
return float(block.max()) < float(thresholds[max(eh, ew)])
|
| 190 |
+
|
| 191 |
+
return _walk_squares(field.shape[0], field.shape[1], field.shape[2], root, keep, edge_rects)
|
| 192 |
+
|
| 193 |
+
|
| 194 |
+
def layout_from_desired(desired: torch.Tensor, root: int = 8, edge_rects: bool = False) -> LotLayout:
|
| 195 |
+
"""Keep a square only when every cell inside allows that extent.
|
| 196 |
+
|
| 197 |
+
``desired`` stores the coarsest square extent each finest cell may belong
|
| 198 |
+
to. This is the merge rule used for mask levels and for blur radii: a
|
| 199 |
+
block is coarsened only when its most demanding cell still allows it.
|
| 200 |
+
"""
|
| 201 |
+
field = _as_time_field(desired).detach().float()
|
| 202 |
+
|
| 203 |
+
def keep(t: int, u: int, v: int, eh: int, ew: int) -> bool:
|
| 204 |
+
block = field[t, u:u + eh, v:v + ew]
|
| 205 |
+
return float(block.min()) >= float(max(eh, ew))
|
| 206 |
+
|
| 207 |
+
return _walk_squares(field.shape[0], field.shape[1], field.shape[2], root, keep, edge_rects)
|
| 208 |
+
|
| 209 |
+
|
| 210 |
+
def levels_to_desired(level: torch.Tensor, max_level: int = 3) -> torch.Tensor:
|
| 211 |
+
"""Map paper levels ``{0,1,2,3}`` to extents ``{8,4,2,1}`` (eq. 19)."""
|
| 212 |
+
if int(level.min()) < 0 or int(level.max()) > max_level:
|
| 213 |
+
raise ValueError(f"levels must lie in 0..{max_level}")
|
| 214 |
+
return 2 ** (max_level - level.to(torch.int64))
|
| 215 |
+
|
| 216 |
+
|
| 217 |
+
def layout_from_regions(
|
| 218 |
+
height: int,
|
| 219 |
+
width: int,
|
| 220 |
+
background_level: int,
|
| 221 |
+
regions: list[tuple[torch.Tensor, int]],
|
| 222 |
+
root: int = 8,
|
| 223 |
+
time: int = 1,
|
| 224 |
+
edge_rects: bool = False,
|
| 225 |
+
) -> LotLayout:
|
| 226 |
+
"""Eq. 19. Overlapping regions keep the finer (larger) level.
|
| 227 |
+
|
| 228 |
+
Each mask is ``(H, W)`` or ``(T, H, W)`` with true on the selected cells.
|
| 229 |
+
The same masks are applied on every frame when ``time > 1`` and the mask
|
| 230 |
+
has no time axis. Per-frame masks must already have length ``time``.
|
| 231 |
+
"""
|
| 232 |
+
level = torch.full((time, height, width), int(background_level), dtype=torch.int64)
|
| 233 |
+
for mask, region_level in regions:
|
| 234 |
+
if region_level <= background_level:
|
| 235 |
+
raise ValueError("selected regions must be finer than the background")
|
| 236 |
+
mask_field = mask.to(dtype=torch.bool)
|
| 237 |
+
if mask_field.ndim == 2:
|
| 238 |
+
mask_field = mask_field.unsqueeze(0).expand(time, -1, -1)
|
| 239 |
+
if tuple(mask_field.shape) != (time, height, width):
|
| 240 |
+
raise ValueError(f"mask shape {tuple(mask_field.shape)} != {(time, height, width)}")
|
| 241 |
+
picked = torch.full_like(level, int(region_level))
|
| 242 |
+
level = torch.where(mask_field, torch.maximum(level, picked), level)
|
| 243 |
+
return layout_from_desired(levels_to_desired(level), root=root, edge_rects=edge_rects)
|
| 244 |
+
|
| 245 |
+
|
| 246 |
+
def shape_features(extent: tuple[int, int, int], include_time: bool = False) -> torch.Tensor:
|
| 247 |
+
"""Eq. 11, optionally followed by ``log2(et)`` for temporal extents."""
|
| 248 |
+
et, eh, ew = extent
|
| 249 |
+
if min(et, eh, ew) < 1:
|
| 250 |
+
raise ValueError(f"bad extent {extent}")
|
| 251 |
+
values = [
|
| 252 |
+
math.log2(eh),
|
| 253 |
+
math.log2(ew),
|
| 254 |
+
math.log2(eh * ew),
|
| 255 |
+
math.log2(eh / ew),
|
| 256 |
+
]
|
| 257 |
+
if include_time:
|
| 258 |
+
values.append(math.log2(et))
|
| 259 |
+
return torch.tensor(values, dtype=torch.float32)
|
| 260 |
+
|
| 261 |
+
|
| 262 |
+
def smooth_detail(detail: torch.Tensor, sigma: float) -> torch.Tensor:
|
| 263 |
+
"""Separable Gaussian on the last two axes. ``sigma <= 0`` is a no-op."""
|
| 264 |
+
if sigma <= 0:
|
| 265 |
+
return detail
|
| 266 |
+
radius = max(1, int(math.ceil(3 * sigma)))
|
| 267 |
+
coords = torch.arange(-radius, radius + 1, dtype=torch.float32, device=detail.device)
|
| 268 |
+
kernel = torch.exp(-0.5 * (coords / float(sigma)) ** 2)
|
| 269 |
+
kernel = kernel / kernel.sum()
|
| 270 |
+
return _conv_last(detail.float(), kernel, radius).to(dtype=detail.dtype)
|
| 271 |
+
|
| 272 |
+
|
| 273 |
+
def _conv_last(field: torch.Tensor, kernel: torch.Tensor, radius: int) -> torch.Tensor:
|
| 274 |
+
"""Gaussian along height, then along width."""
|
| 275 |
+
flat = field.reshape(-1, 1, field.shape[-2], field.shape[-1])
|
| 276 |
+
weight_w = kernel.view(1, 1, 1, -1)
|
| 277 |
+
weight_h = kernel.view(1, 1, -1, 1)
|
| 278 |
+
flat = torch.nn.functional.pad(flat, (radius, radius, 0, 0), mode="reflect")
|
| 279 |
+
flat = torch.nn.functional.conv2d(flat, weight_w)
|
| 280 |
+
flat = torch.nn.functional.pad(flat, (0, 0, radius, radius), mode="reflect")
|
| 281 |
+
flat = torch.nn.functional.conv2d(flat, weight_h)
|
| 282 |
+
return flat.reshape(field.shape).to(dtype=field.dtype)
|
| 283 |
+
|
| 284 |
+
|
| 285 |
+
def prepare_detail(scores: torch.Tensor, gain: float = 1.0, sigma: float = 0.0) -> torch.Tensor:
|
| 286 |
+
"""Gain, optional smooth, clip to ``[0, 1]``. Scores are on the token lattice."""
|
| 287 |
+
return smooth_detail(scores.float() * float(gain), sigma).clamp(0, 1)
|
| 288 |
+
|
| 289 |
+
|
| 290 |
+
def area_average(pixel_scores: torch.Tensor, token_height: int, token_width: int) -> torch.Tensor:
|
| 291 |
+
"""Area-average a pixel detail map onto the uniform token lattice."""
|
| 292 |
+
if pixel_scores.ndim == 2:
|
| 293 |
+
field = pixel_scores[None, None]
|
| 294 |
+
squeeze = 2
|
| 295 |
+
elif pixel_scores.ndim == 3:
|
| 296 |
+
field = pixel_scores[:, None]
|
| 297 |
+
squeeze = 1
|
| 298 |
+
else:
|
| 299 |
+
raise ValueError("pixel scores must be (H, W) or (T, H, W)")
|
| 300 |
+
pooled = torch.nn.functional.adaptive_avg_pool2d(field.float(), (token_height, token_width))
|
| 301 |
+
if squeeze == 2:
|
| 302 |
+
return pooled[0, 0]
|
| 303 |
+
return pooled[:, 0]
|
| 304 |
+
|
| 305 |
+
|
| 306 |
+
def blur_radius(depth: torch.Tensor, focal_depth: float, r_target: float) -> torch.Tensor:
|
| 307 |
+
"""Circle-of-confusion radius, eqs. 21–22.
|
| 308 |
+
|
| 309 |
+
``K`` is set so the 95th percentile of absolute inverse-depth deviation
|
| 310 |
+
equals ``r_target``. Depth must be positive.
|
| 311 |
+
"""
|
| 312 |
+
if focal_depth <= 0 or r_target < 0:
|
| 313 |
+
raise ValueError("focal depth and r_target must be positive")
|
| 314 |
+
deviation = (depth.float().clamp(min=1e-8).reciprocal() - (1.0 / float(focal_depth))).abs()
|
| 315 |
+
flat = deviation.reshape(-1)
|
| 316 |
+
percentile = torch.quantile(flat, 0.95).clamp(min=1e-8)
|
| 317 |
+
return deviation * (float(r_target) / percentile)
|
| 318 |
+
|
| 319 |
+
|
| 320 |
+
def layout_from_blur(radius: torch.Tensor, support: dict[int, float], root: int = 8,
|
| 321 |
+
edge_rects: bool = False) -> LotLayout:
|
| 322 |
+
"""Merge a block only when its smallest blur radius supports that extent.
|
| 323 |
+
|
| 324 |
+
``support[b]`` is the minimum radius that still permits an extent-``b``
|
| 325 |
+
token. ``support`` values must increase with ``b``. Cells with a small
|
| 326 |
+
radius (in focus) force the block to split.
|
| 327 |
+
"""
|
| 328 |
+
sizes = [size for size in (8, 4, 2) if size <= root]
|
| 329 |
+
missing = [size for size in sizes if size not in support]
|
| 330 |
+
if missing:
|
| 331 |
+
raise ValueError(f"blur support missing extents {missing}")
|
| 332 |
+
ordered = [float(support[size]) for size in sizes]
|
| 333 |
+
if ordered != sorted(ordered, reverse=True):
|
| 334 |
+
raise ValueError("blur support must be larger for coarser extents")
|
| 335 |
+
field = _as_time_field(radius).detach().float()
|
| 336 |
+
|
| 337 |
+
def keep(t: int, u: int, v: int, eh: int, ew: int) -> bool:
|
| 338 |
+
block = field[t, u:u + eh, v:v + ew]
|
| 339 |
+
return float(block.min()) >= float(support[max(eh, ew)])
|
| 340 |
+
|
| 341 |
+
return _walk_squares(field.shape[0], field.shape[1], field.shape[2], root, keep, edge_rects)
|
| 342 |
+
|
| 343 |
+
|
| 344 |
+
def directional_variation(luminance: torch.Tensor) -> torch.Tensor:
|
| 345 |
+
"""Per-cell max of absolute forward differences, padded at the far edge.
|
| 346 |
+
|
| 347 |
+
This is the error field fed to the VRS tolerance in Appendix A.3. The
|
| 348 |
+
paper cites Yang et al. 2019 for the shading rule and does not publish
|
| 349 |
+
the finite-difference stencil; the forward difference is the stencil
|
| 350 |
+
used here.
|
| 351 |
+
"""
|
| 352 |
+
field = _as_time_field(luminance).float()
|
| 353 |
+
dx = (field[..., :, 1:] - field[..., :, :-1]).abs()
|
| 354 |
+
dy = (field[..., 1:, :] - field[..., :-1, :]).abs()
|
| 355 |
+
dx = torch.nn.functional.pad(dx, (0, 1))
|
| 356 |
+
dy = torch.nn.functional.pad(dy, (0, 0, 0, 1))
|
| 357 |
+
return torch.maximum(dx, dy).reshape(luminance.shape).to(dtype=luminance.dtype)
|
| 358 |
+
|
| 359 |
+
|
| 360 |
+
def layout_from_vrs(
|
| 361 |
+
luminance: torch.Tensor,
|
| 362 |
+
sensitivity: float,
|
| 363 |
+
ambient: float,
|
| 364 |
+
root: int = 8,
|
| 365 |
+
edge_rects: bool = False,
|
| 366 |
+
) -> LotLayout:
|
| 367 |
+
"""Coarsen a block when its variation stays under ``s * (mean(I) + a)``.
|
| 368 |
+
|
| 369 |
+
Training in the paper draws ``s`` from ``[0.01, 0.1]`` and ``a`` from
|
| 370 |
+
``[0.01, 1]``. Larger values permit coarser tokens. The decision is
|
| 371 |
+
square, on the same quadtree as the detail maps. Directional rectangles
|
| 372 |
+
can still be passed through ``layout_from_rects``.
|
| 373 |
+
"""
|
| 374 |
+
if sensitivity < 0 or ambient < 0:
|
| 375 |
+
raise ValueError("VRS sensitivity and ambient must be non-negative")
|
| 376 |
+
luma = _as_time_field(luminance).float()
|
| 377 |
+
error = _as_time_field(directional_variation(luminance)).float()
|
| 378 |
+
|
| 379 |
+
def keep(t: int, u: int, v: int, eh: int, ew: int) -> bool:
|
| 380 |
+
block_luma = luma[t, u:u + eh, v:v + ew]
|
| 381 |
+
block_error = error[t, u:u + eh, v:v + ew]
|
| 382 |
+
tau = float(sensitivity) * (float(block_luma.mean()) + float(ambient))
|
| 383 |
+
return float(block_error.max()) < tau
|
| 384 |
+
|
| 385 |
+
return _walk_squares(luma.shape[0], luma.shape[1], luma.shape[2], root, keep, edge_rects)
|
lot/layout_sources.py
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Content-aware LoT layouts: semantic mask, bounding boxes, depth of field, texture.
|
| 2 |
+
|
| 3 |
+
The LoT paper's supplementary (layout-adaptive generation) derives per-frame
|
| 4 |
+
layouts from these four sources with one trained model: fine tokens on the
|
| 5 |
+
primary action region, coarser ones on secondary objects and background. Each
|
| 6 |
+
source here becomes a per-token *desired extent* map (the coarsest square a
|
| 7 |
+
token may join: 1, 2 or 4), and ``layout_from_desired_any`` turns that into a
|
| 8 |
+
quadtree layout per latent frame, so the layout follows the content over time.
|
| 9 |
+
|
| 10 |
+
Pixel inputs are ``(F, H, W)`` (or ``(H, W)`` for a single frame) in [0, 1] at
|
| 11 |
+
pixel resolution; they are area-averaged onto the token lattice (one H3 token =
|
| 12 |
+
32×32 pixels) and mapped from pixel frames onto H3's latent frames
|
| 13 |
+
(``(1, 4, 4, 4, 4)`` pixel frames per latent frame, repeating).
|
| 14 |
+
"""
|
| 15 |
+
|
| 16 |
+
from __future__ import annotations
|
| 17 |
+
|
| 18 |
+
import torch
|
| 19 |
+
import torch.nn.functional as F
|
| 20 |
+
|
| 21 |
+
from h3_positions import FRAME_PER_TOKEN
|
| 22 |
+
from layout import LotLayout, TokenRect, area_average, blur_radius, layout_from_desired
|
| 23 |
+
|
| 24 |
+
EXTENTS = (1, 2, 4)
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def latent_frame_spans(latent_t: int) -> list[tuple[int, int]]:
|
| 28 |
+
"""``[start, stop)`` pixel frames covered by each H3 latent frame."""
|
| 29 |
+
spans, start = [], 0
|
| 30 |
+
for k in range(latent_t):
|
| 31 |
+
n = FRAME_PER_TOKEN[k % 5]
|
| 32 |
+
spans.append((start, start + n))
|
| 33 |
+
start += n
|
| 34 |
+
return spans
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def to_latent_frames(field: torch.Tensor, latent_t: int, reduce: str = "max") -> torch.Tensor:
|
| 38 |
+
"""``(F, h, w)`` per pixel frame -> ``(latent_t, h, w)`` per latent frame.
|
| 39 |
+
|
| 40 |
+
A single frame is broadcast. Any other frame count is stretched onto the
|
| 41 |
+
expected pixel-frame count first, so a mask from a shorter or longer clip
|
| 42 |
+
still lines up in time.
|
| 43 |
+
"""
|
| 44 |
+
if field.ndim == 2:
|
| 45 |
+
field = field[None]
|
| 46 |
+
spans = latent_frame_spans(latent_t)
|
| 47 |
+
total = spans[-1][1]
|
| 48 |
+
if field.shape[0] == 1:
|
| 49 |
+
return field.expand(latent_t, *field.shape[1:]).clone()
|
| 50 |
+
if field.shape[0] != total:
|
| 51 |
+
index = torch.linspace(0, field.shape[0] - 1, total).round().long()
|
| 52 |
+
field = field[index]
|
| 53 |
+
pool = (lambda x: x.amax(0)) if reduce == "max" else (lambda x: x.mean(0))
|
| 54 |
+
return torch.stack([pool(field[a:b]) for a, b in spans])
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def layout_from_desired_any(desired: torch.Tensor) -> LotLayout:
|
| 58 |
+
"""Quadtree (4 → 2 → 1) for any ``(T, H, W)`` grid.
|
| 59 |
+
|
| 60 |
+
A 4×4 cell cut by the far edge keeps the clipped rectangle when its cells
|
| 61 |
+
allow it (4×2 on the 2-token strip of H3's 24×42, 2×4 on a 2-row strip), so
|
| 62 |
+
edges coarsen like the interior instead of falling back to 1×1. Every
|
| 63 |
+
clipped extent is in H3's bank (FINE + MID + COARSE).
|
| 64 |
+
"""
|
| 65 |
+
return layout_from_desired(desired.long().clamp(1, 4), root=4, edge_rects=True)
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
def desired_from_levels(fine: torch.Tensor, mid: torch.Tensor | None, background: int) -> torch.Tensor:
|
| 69 |
+
"""Boolean token maps -> desired extents: fine 1, mid 2, everything else ``background``."""
|
| 70 |
+
if background not in EXTENTS:
|
| 71 |
+
raise ValueError(f"background extent must be one of {EXTENTS}")
|
| 72 |
+
desired = torch.full(fine.shape, background, dtype=torch.long)
|
| 73 |
+
if mid is not None:
|
| 74 |
+
desired[mid & (desired > 2)] = 2
|
| 75 |
+
desired[fine] = 1
|
| 76 |
+
return desired
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
def _tokens(pixel: torch.Tensor, latent_t: int, token_h: int, token_w: int, reduce: str = "max") -> torch.Tensor:
|
| 80 |
+
pixel = pixel.float()
|
| 81 |
+
if pixel.ndim == 2:
|
| 82 |
+
pixel = pixel[None]
|
| 83 |
+
return to_latent_frames(area_average(pixel, token_h, token_w), latent_t, reduce)
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
def _dilate(field: torch.Tensor, tokens: int) -> torch.Tensor:
|
| 87 |
+
if tokens <= 0:
|
| 88 |
+
return field
|
| 89 |
+
k = 2 * tokens + 1
|
| 90 |
+
return F.max_pool2d(field.float()[:, None], k, stride=1, padding=tokens)[:, 0] > 0.5
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def mask_layout(mask: torch.Tensor, latent_t: int, token_h: int, token_w: int, *,
|
| 94 |
+
secondary: torch.Tensor | None = None, background: int = 4,
|
| 95 |
+
coverage: float = 0.05, dilate: int = 1) -> LotLayout:
|
| 96 |
+
"""Semantic mask (or rasterised boxes): 1×1 on the mask, 2×2 on ``secondary``, ``background`` elsewhere.
|
| 97 |
+
|
| 98 |
+
A token counts as covered when ``coverage`` of its 32×32 pixels are inside
|
| 99 |
+
the mask; ``dilate`` grows the fine region by that many tokens so edges stay sharp.
|
| 100 |
+
"""
|
| 101 |
+
fine = _dilate(_tokens(mask, latent_t, token_h, token_w) > coverage, dilate)
|
| 102 |
+
mid = None
|
| 103 |
+
if secondary is not None:
|
| 104 |
+
mid = _dilate(_tokens(secondary, latent_t, token_h, token_w) > coverage, dilate)
|
| 105 |
+
return layout_from_desired_any(desired_from_levels(fine, mid, background))
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def boxes_to_mask(boxes: list[tuple[float, float, float, float]], height: int, width: int,
|
| 109 |
+
frames: int = 1) -> torch.Tensor:
|
| 110 |
+
"""Normalised ``(x0, y0, x1, y1)`` boxes -> ``(frames, height, width)`` mask (same boxes every frame)."""
|
| 111 |
+
mask = torch.zeros(frames, height, width)
|
| 112 |
+
for x0, y0, x1, y1 in boxes:
|
| 113 |
+
a, b = sorted((x0, x1))
|
| 114 |
+
c, d = sorted((y0, y1))
|
| 115 |
+
mask[:, int(c * height):max(int(d * height), int(c * height) + 1),
|
| 116 |
+
int(a * width):max(int(b * width), int(a * width) + 1)] = 1.0
|
| 117 |
+
return mask
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
# lot/layout.py layout_from_blur. A cell may join extent b only when its circle of
|
| 121 |
+
# confusion is at least this radius. Coarser extents need a larger defocus.
|
| 122 |
+
_BLUR_SUPPORT = ((8, 3.0), (4, 1.0), (2, 0.2))
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
def _desired_from_radius(radius: torch.Tensor, background: int) -> torch.Tensor:
|
| 126 |
+
"""Blur radius -> coarsest extent each cell may join. In-focus cells stay at 1.
|
| 127 |
+
|
| 128 |
+
``background`` is the coarsest square this caller can emit. H3's video heads stop at 4,
|
| 129 |
+
so a radius that would be an 8×8 square stays at 4.
|
| 130 |
+
"""
|
| 131 |
+
desired = torch.ones(radius.shape, dtype=torch.long)
|
| 132 |
+
for extent, support in reversed(_BLUR_SUPPORT):
|
| 133 |
+
if extent > background:
|
| 134 |
+
continue
|
| 135 |
+
desired = torch.where(radius >= support, torch.full_like(desired, extent), desired)
|
| 136 |
+
return desired
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
def depth_layout(depth: torch.Tensor, latent_t: int, token_h: int, token_w: int, *,
|
| 140 |
+
focus: float = 0.75, band: float = 0.15, background: int = 4,
|
| 141 |
+
r_target: float = 4.0) -> LotLayout:
|
| 142 |
+
"""Derived depth-of-field layout. Fine tokens on the focal plane, coarse squares on the defocus.
|
| 143 |
+
|
| 144 |
+
The radius is the circle of confusion (eqs. 21–22): ``|1/z - 1/z_focus|``, scaled so the
|
| 145 |
+
95th percentile of each latent frame equals ``r_target``. A square is kept only when every
|
| 146 |
+
cell inside it is blurred enough for that extent. ``depth`` is non-negative and on the same
|
| 147 |
+
scale as ``focus``. A Depth Anything map (brighter = nearer, typically in [0, 1]) is legal
|
| 148 |
+
when ``focus`` is read from that same map. ``band`` stays in the signature for existing
|
| 149 |
+
callers; the radius replaces that fixed band.
|
| 150 |
+
"""
|
| 151 |
+
if background not in EXTENTS:
|
| 152 |
+
raise ValueError(f"background extent must be one of {EXTENTS}")
|
| 153 |
+
if focus <= 0 or r_target < 0 or band <= 0:
|
| 154 |
+
raise ValueError("focus must be positive, on the depth map's scale; r_target and band must be positive")
|
| 155 |
+
tokens = _tokens(depth, latent_t, token_h, token_w, reduce="mean")
|
| 156 |
+
if not torch.isfinite(tokens).all() or float(tokens.min()) < 0:
|
| 157 |
+
raise ValueError("depth must be finite and non-negative")
|
| 158 |
+
desired = torch.stack([_desired_from_radius(blur_radius(frame, float(focus), r_target), background)
|
| 159 |
+
for frame in tokens])
|
| 160 |
+
return layout_from_desired_any(desired)
|
| 161 |
+
|
| 162 |
+
|
| 163 |
+
def texture_scores(image: torch.Tensor, token_h: int, token_w: int) -> torch.Tensor:
|
| 164 |
+
"""Per-token texture: std of luminance inside each token's pixels, ``(F, token_h, token_w)``.
|
| 165 |
+
|
| 166 |
+
``image`` is ``(F, H, W, C)`` or ``(F, H, W)`` in [0, 1].
|
| 167 |
+
"""
|
| 168 |
+
if image.ndim == 4:
|
| 169 |
+
image = (image[..., :3].float() * torch.tensor([0.299, 0.587, 0.114])).sum(-1)
|
| 170 |
+
lum = image.float()
|
| 171 |
+
mean = area_average(lum, token_h, token_w)
|
| 172 |
+
sq = area_average(lum * lum, token_h, token_w)
|
| 173 |
+
return (sq - mean * mean).clamp(min=0).sqrt()
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
def score_layout(score: torch.Tensor, *, fine_q: float = 0.7, mid_q: float = 0.4,
|
| 177 |
+
background: int = 4) -> LotLayout:
|
| 178 |
+
"""``(T, H, W)`` importance scores -> layout by per-frame quantiles: top ``1-fine_q`` 1×1, next band 2×2."""
|
| 179 |
+
flat = score.flatten(1)
|
| 180 |
+
hi = flat.quantile(fine_q, dim=1)[:, None, None]
|
| 181 |
+
lo = flat.quantile(mid_q, dim=1)[:, None, None]
|
| 182 |
+
# Strictly above the low quantile: perfectly flat tokens (ties at the bottom) may go coarse.
|
| 183 |
+
mid = score > lo
|
| 184 |
+
return layout_from_desired_any(desired_from_levels((score >= hi) & mid, mid, background))
|
| 185 |
+
|
| 186 |
+
|
| 187 |
+
def texture_layout(image: torch.Tensor, latent_t: int, token_h: int, token_w: int, *,
|
| 188 |
+
fine_q: float = 0.7, mid_q: float = 0.4, background: int = 4) -> LotLayout:
|
| 189 |
+
"""Texture variance (the paper's VRS source): fine where the image is busy, coarse where it is flat."""
|
| 190 |
+
score = to_latent_frames(texture_scores(image, token_h, token_w), latent_t)
|
| 191 |
+
return score_layout(score, fine_q=fine_q, mid_q=mid_q, background=background)
|
| 192 |
+
|
| 193 |
+
|
| 194 |
+
def compression(layout: LotLayout) -> float:
|
| 195 |
+
return layout.dense_count / max(layout.count, 1)
|
| 196 |
+
|
| 197 |
+
|
| 198 |
+
__all__ = [
|
| 199 |
+
"EXTENTS", "TokenRect", "boxes_to_mask", "compression", "depth_layout", "desired_from_levels",
|
| 200 |
+
"latent_frame_spans", "layout_from_desired_any", "mask_layout", "score_layout", "texture_layout",
|
| 201 |
+
"texture_scores", "to_latent_frames",
|
| 202 |
+
]
|
lot/lot_paths.py
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Machine-specific paths for the LoT scripts, each overridable by an environment variable.
|
| 2 |
+
|
| 3 |
+
The defaults are the machine these scripts were developed on. On another machine,
|
| 4 |
+
export the variables instead of editing the scripts:
|
| 5 |
+
|
| 6 |
+
| Variable | What |
|
| 7 |
+
|---|---|
|
| 8 |
+
| ``MUSUBI_SRC`` | musubi-tuner ``src`` directory (H3 model, int8 loader, LoRA, VAE); defaults to ``../third_party`` when vendored |
|
| 9 |
+
| ``LOT_H3_CHECKPOINT`` | pruned int8 ConvRot FL2VA DiT the adapters were trained on (``hf download Comfy-Org/MiniMax-H3 diffusion_models/minimax_h3_fl2va_pruned_int8_convrot.safetensors``) |
|
| 10 |
+
| ``LOT_H3_VAE`` | H3 video VAE (fp16) |
|
| 11 |
+
| ``LOT_H3_TEXT_ENCODER`` | Qwen3-VL-32B MiniMax text encoder (only for caching captions) |
|
| 12 |
+
| ``LOT_H3_STILLS_CACHE`` | H3 still cache with ``_te`` text (default training/render prompts) |
|
| 13 |
+
| ``LOT_H3_STILLS_CAPTIONS`` | caption ``.txt`` files for that cache |
|
| 14 |
+
"""
|
| 15 |
+
|
| 16 |
+
from __future__ import annotations
|
| 17 |
+
|
| 18 |
+
import os
|
| 19 |
+
from pathlib import Path
|
| 20 |
+
|
| 21 |
+
# A vendored subset of musubi-tuner (``../third_party``) wins over this machine's checkout.
|
| 22 |
+
_VENDORED = Path(__file__).resolve().parent.parent / "third_party"
|
| 23 |
+
MUSUBI_SRC = os.environ.get(
|
| 24 |
+
"MUSUBI_SRC", str(_VENDORED) if (_VENDORED / "musubi_tuner").is_dir() else "/media/2TB/musubi-tuner/src")
|
| 25 |
+
LOT_H3_CHECKPOINT = Path(os.environ.get(
|
| 26 |
+
"LOT_H3_CHECKPOINT",
|
| 27 |
+
"/media/2TB/models/minimax_h3/minimax_h3_fl2va_pruned_int8_convrot.safetensors"))
|
| 28 |
+
LOT_H3_VAE = Path(os.environ.get(
|
| 29 |
+
"LOT_H3_VAE", "/media/2TB/ComfyUI/models/vae/minimax_h3_video_vae_fp16.safetensors"))
|
| 30 |
+
LOT_H3_TEXT_ENCODER = Path(os.environ.get(
|
| 31 |
+
"LOT_H3_TEXT_ENCODER",
|
| 32 |
+
"/media/2TB/minimax-h3-nvfp4/text_encoders/qwen3vl_32b_minimax_h3_nvfp4_awq.safetensors"))
|
| 33 |
+
LOT_H3_STILLS_CACHE = Path(os.environ.get(
|
| 34 |
+
"LOT_H3_STILLS_CACHE", "/media/2TB/lora-data/h3_lot/cache_iso3d"))
|
| 35 |
+
LOT_H3_STILLS_CAPTIONS = Path(os.environ.get(
|
| 36 |
+
"LOT_H3_STILLS_CAPTIONS", "/media/2TB/lora-data/h3_lot/isometric_3d_stills"))
|
lot/mixed_res.py
ADDED
|
@@ -0,0 +1,513 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Mixed-resolution additions to LoT, adapted from MPEG-Diffusion (Dennis Menn, Tianpei Gu, Guande He, Aryan V S, Yanke Song, Nan Liu, Desai Xie, Xun Huang, Chenfeng Xu and Diana Marculescu, 2026).
|
| 2 |
+
|
| 3 |
+
MPEG-Diffusion: Mixed-Resolution Video Generation, Dennis Menn, Tianpei Gu, Guande He, Aryan V S, Yanke Song, Nan Liu,
|
| 4 |
+
Desai Xie, Xun Huang, Chenfeng Xu and Diana Marculescu (UT Austin, Roblox), arXiv:2611.12069. That paper
|
| 5 |
+
keeps two token grids (HR inside a mask, LR elsewhere). LoT has one sequence of
|
| 6 |
+
mixed extents. The three pieces here carry its ideas over to that sequence:
|
| 7 |
+
|
| 8 |
+
1. ``DualKV`` (their dual-resolution attention, Sec. 3). Their Proposition 1:
|
| 9 |
+
with one shared K/V, a coarse token can stand in for ``n`` fine tokens under
|
| 10 |
+
softmax only when those fine tokens are identical. So 1x1 queries here attend
|
| 11 |
+
to a *dense* key/value view: every coarse token's key and value are expanded
|
| 12 |
+
to its ``n`` cells by ``f_up(k) = k + U V^T k`` (their "linear delta":
|
| 13 |
+
exact copy plus a rank-``r`` correction, zero at init), before RoPE, and the
|
| 14 |
+
expanded keys are rotated at the cells' dense positions. Every other query
|
| 15 |
+
(text, audio, coarse tokens) keeps LoT's native attention, which already sees
|
| 16 |
+
each fine token individually, so no ``f_down`` is needed. With zero deltas a
|
| 17 |
+
fine query sees a coarse token as ``n`` copies at its cells.
|
| 18 |
+
2. ``RegionHead`` (their self-selected HR region, Sec. 3.4): ``s = w^T LN(h)``
|
| 19 |
+
on a middle block's video tokens, smoothed in space and time, thresholded at
|
| 20 |
+
``m + alpha (p - m)`` per video (median ``m``, 99th percentile ``p``), all
|
| 21 |
+
coarse when ``p`` is under a floor. The mask becomes a per-latent-frame LoT
|
| 22 |
+
layout. Labels here are self-supervised in latent space (``region_labels``):
|
| 23 |
+
tokens where the coarse layout's clean estimate is furthest from the dense
|
| 24 |
+
model's, a latent stand-in for their per-patch LPIPS label.
|
| 25 |
+
3. ``SeamMixer`` (their mixed-resolution decoder's mixer ``G``, eq. 6): a small
|
| 26 |
+
3D-conv residual on the final latent, applied outside the fine region only,
|
| 27 |
+
zero-initialised so it starts as the identity. LoT already returns a dense
|
| 28 |
+
latent, so there is no upscaler; the residual targets coarse-token blocking
|
| 29 |
+
and the seams between extents.
|
| 30 |
+
|
| 31 |
+
Nothing here imports model code; ``h3_musubi`` wires it into H3's forward.
|
| 32 |
+
"""
|
| 33 |
+
|
| 34 |
+
from __future__ import annotations
|
| 35 |
+
|
| 36 |
+
import contextlib
|
| 37 |
+
import math
|
| 38 |
+
import types
|
| 39 |
+
|
| 40 |
+
import torch
|
| 41 |
+
import torch.nn as nn
|
| 42 |
+
import torch.nn.functional as F
|
| 43 |
+
|
| 44 |
+
from layout import LotLayout, smooth_detail
|
| 45 |
+
from layout_sources import layout_from_desired_any
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
# --------------------------------------------------------------------- 2. dual K/V
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
class DualKVPlan:
|
| 52 |
+
"""Index tensors for one layout: which video rows are 1x1, and how to assemble the dense view.
|
| 53 |
+
|
| 54 |
+
The expanded order is: the 1x1 tokens (in layout order), then for each larger extent its
|
| 55 |
+
tokens, each followed by its ``eh * ew`` cells in row-major order. ``dense_order`` permutes
|
| 56 |
+
that into dense ``(t, u, v)`` order, the order of H3's dense video positions.
|
| 57 |
+
"""
|
| 58 |
+
|
| 59 |
+
def __init__(self, layout: LotLayout):
|
| 60 |
+
self.layout = layout
|
| 61 |
+
fine, groups = [], []
|
| 62 |
+
cells = []
|
| 63 |
+
for index, rect in enumerate(layout.rects):
|
| 64 |
+
if rect.extent == (1, 1, 1):
|
| 65 |
+
fine.append(index)
|
| 66 |
+
width, plane = layout.width, layout.height * layout.width
|
| 67 |
+
for index in fine:
|
| 68 |
+
rect = layout.rects[index]
|
| 69 |
+
cells.append(rect.t * plane + rect.u * width + rect.v)
|
| 70 |
+
for extent, indices in layout.groups():
|
| 71 |
+
if extent == (1, 1, 1):
|
| 72 |
+
continue
|
| 73 |
+
groups.append((extent, torch.tensor(indices, dtype=torch.long)))
|
| 74 |
+
for index in indices:
|
| 75 |
+
rect = layout.rects[index]
|
| 76 |
+
for du in range(rect.eh):
|
| 77 |
+
for dv in range(rect.ew):
|
| 78 |
+
cells.append(rect.t * plane + (rect.u + du) * width + rect.v + dv)
|
| 79 |
+
cells = torch.tensor(cells, dtype=torch.long)
|
| 80 |
+
if cells.numel() != layout.dense_count or torch.unique(cells).numel() != layout.dense_count:
|
| 81 |
+
raise ValueError("layout does not cover the dense grid exactly once")
|
| 82 |
+
self.fine_rows = torch.tensor(fine, dtype=torch.long)
|
| 83 |
+
fine_mask = torch.zeros(layout.count, dtype=torch.bool)
|
| 84 |
+
fine_mask[self.fine_rows] = True
|
| 85 |
+
self.other_rows = torch.nonzero(~fine_mask).squeeze(1)
|
| 86 |
+
self.groups = groups
|
| 87 |
+
self.dense_order = torch.argsort(cells)
|
| 88 |
+
|
| 89 |
+
def to(self, device) -> "DualKVPlan":
|
| 90 |
+
self.fine_rows = self.fine_rows.to(device)
|
| 91 |
+
self.other_rows = self.other_rows.to(device)
|
| 92 |
+
self.dense_order = self.dense_order.to(device)
|
| 93 |
+
self.groups = [(extent, rows.to(device)) for extent, rows in self.groups]
|
| 94 |
+
return self
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
def _key(layer: int, extent: tuple[int, int, int], which: str) -> str:
|
| 98 |
+
return f"l{layer}_{extent[1]}x{extent[2]}_{which}"
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
class UpMaps(nn.Module):
|
| 102 |
+
"""``f_up`` per block, extent, and K or V: ``k -> k + (k V) U`` reshaped to ``n`` cells.
|
| 103 |
+
|
| 104 |
+
``V`` is ``(d, r)``, ``U`` is ``(r, n * d)``, shared by all heads of a block (the paper fits
|
| 105 |
+
one map per layer). ``U`` starts at zero, so the map starts as an exact copy.
|
| 106 |
+
"""
|
| 107 |
+
|
| 108 |
+
def __init__(self, layers: int, head_dim: int, extents, rank: int = 16):
|
| 109 |
+
super().__init__()
|
| 110 |
+
self.head_dim, self.rank = int(head_dim), int(rank)
|
| 111 |
+
self.extents = [tuple(map(int, e)) for e in extents if tuple(e) != (1, 1, 1)]
|
| 112 |
+
self.down = nn.ParameterDict()
|
| 113 |
+
self.up = nn.ParameterDict()
|
| 114 |
+
for layer in range(layers):
|
| 115 |
+
for extent in self.extents:
|
| 116 |
+
cells = extent[1] * extent[2]
|
| 117 |
+
for which in ("k", "v"):
|
| 118 |
+
name = _key(layer, extent, which)
|
| 119 |
+
down = torch.empty(self.head_dim, self.rank)
|
| 120 |
+
nn.init.normal_(down, std=1.0 / math.sqrt(self.head_dim))
|
| 121 |
+
self.down[name] = nn.Parameter(down)
|
| 122 |
+
self.up[name] = nn.Parameter(torch.zeros(self.rank, cells * self.head_dim))
|
| 123 |
+
|
| 124 |
+
def expand(self, layer: int, extent: tuple[int, int, int], which: str, rows: torch.Tensor) -> torch.Tensor:
|
| 125 |
+
"""``(B, N, H, d)`` -> ``(B, N * n, H, d)``: each token, then its cells row-major."""
|
| 126 |
+
name = _key(layer, extent, which)
|
| 127 |
+
cells = extent[1] * extent[2]
|
| 128 |
+
batch, count, heads, dim = rows.shape
|
| 129 |
+
delta = (rows @ self.down[name].to(rows.dtype)) @ self.up[name].to(rows.dtype) # (B, N, H, n*d)
|
| 130 |
+
delta = delta.reshape(batch, count, heads, cells, dim).transpose(2, 3) # (B, N, n, H, d)
|
| 131 |
+
expanded = rows.unsqueeze(2) + delta
|
| 132 |
+
return expanded.reshape(batch, count * cells, heads, dim)
|
| 133 |
+
|
| 134 |
+
@torch.no_grad()
|
| 135 |
+
def fit_(self, layer: int, extent: tuple[int, int, int], which: str,
|
| 136 |
+
coarse: torch.Tensor, cells: torch.Tensor, ridge: float = 1e-2) -> float:
|
| 137 |
+
"""Ridge fit of the delta (their init), truncated to rank ``r`` by SVD.
|
| 138 |
+
|
| 139 |
+
``coarse`` is ``(N, d)`` keys (or values) of coarse tokens, ``cells`` is ``(N, n, d)`` the
|
| 140 |
+
dense model's keys at the cells they cover. Solves ``min |coarse W - (cells - coarse)|^2 +
|
| 141 |
+
ridge |W|^2`` for ``W`` ``(d, n*d)``, keeps the top ``r`` singular directions. Returns
|
| 142 |
+
the relative residual of the copy-plus-delta fit."""
|
| 143 |
+
name = _key(layer, extent, which)
|
| 144 |
+
x = coarse.float()
|
| 145 |
+
target = (cells.float() - x.unsqueeze(1)).reshape(x.shape[0], -1)
|
| 146 |
+
self.fit_moments_(layer, extent, which, x.T @ x, x.T @ target, x.shape[0], ridge)
|
| 147 |
+
fitted = x.unsqueeze(1) + ((x @ self.down[name].float()) @ self.up[name].float()).reshape_as(cells)
|
| 148 |
+
return float((fitted - cells.float()).norm() / cells.float().norm().clamp(min=1e-8))
|
| 149 |
+
|
| 150 |
+
@torch.no_grad()
|
| 151 |
+
def fit_moments_(self, layer: int, extent: tuple[int, int, int], which: str,
|
| 152 |
+
xtx: torch.Tensor, xty: torch.Tensor, count: int, ridge: float = 1e-2) -> None:
|
| 153 |
+
"""``fit_`` from accumulated ``X^T X`` ``(d, d)`` and ``X^T (cells - X)`` ``(d, n*d)`` over ``count`` rows."""
|
| 154 |
+
name = _key(layer, extent, which)
|
| 155 |
+
gram = xtx.float() + ridge * count * torch.eye(xtx.shape[0], device=xtx.device)
|
| 156 |
+
weight = torch.linalg.solve(gram, xty.float()) # (d, n*d)
|
| 157 |
+
u, s, vh = torch.linalg.svd(weight, full_matrices=False)
|
| 158 |
+
r = self.rank
|
| 159 |
+
self.down[name].copy_((u[:, :r] * s[:r].sqrt()).to(self.down[name].dtype))
|
| 160 |
+
self.up[name].copy_((s[:r].sqrt().unsqueeze(1) * vh[:r]).to(self.up[name].dtype))
|
| 161 |
+
|
| 162 |
+
|
| 163 |
+
class DualKV:
|
| 164 |
+
"""``UpMaps`` plus a per-layout plan cache; what ``h3_musubi.predict(..., dual=)`` takes."""
|
| 165 |
+
|
| 166 |
+
def __init__(self, maps: UpMaps):
|
| 167 |
+
self.maps = maps
|
| 168 |
+
self._plans: dict = {}
|
| 169 |
+
|
| 170 |
+
def plan(self, layout: LotLayout, device) -> DualKVPlan:
|
| 171 |
+
key = (layout, str(device))
|
| 172 |
+
if key not in self._plans:
|
| 173 |
+
if len(self._plans) > 8:
|
| 174 |
+
self._plans.clear()
|
| 175 |
+
self._plans[key] = DualKVPlan(layout).to(device)
|
| 176 |
+
return self._plans[key]
|
| 177 |
+
|
| 178 |
+
|
| 179 |
+
def _rope(states: torch.Tensor, table: torch.Tensor) -> torch.Tensor:
|
| 180 |
+
"""H3's split-half RoPE (``model._apply_rope_split_half``), repeated so this file needs no model import."""
|
| 181 |
+
pairs = table.shape[-3]
|
| 182 |
+
a, b = states[..., :pairs], states[..., pairs:2 * pairs]
|
| 183 |
+
first = table[..., 0, 0] * a + table[..., 0, 1] * b # element-wise: no broadcast matmul over heads
|
| 184 |
+
second = table[..., 1, 0] * a + table[..., 1, 1] * b
|
| 185 |
+
return torch.cat((first, second, states[..., 2 * pairs:]), dim=-1)
|
| 186 |
+
|
| 187 |
+
|
| 188 |
+
def _sdpa(query: torch.Tensor, key: torch.Tensor, value: torch.Tensor) -> torch.Tensor:
|
| 189 |
+
"""``(B, L, H, d)`` in, ``(B, L, H * d)`` out."""
|
| 190 |
+
out = F.scaled_dot_product_attention(query.transpose(1, 2), key.transpose(1, 2), value.transpose(1, 2))
|
| 191 |
+
return out.transpose(1, 2).reshape(query.shape[0], query.shape[1], -1)
|
| 192 |
+
|
| 193 |
+
|
| 194 |
+
def dual_attention(attn, hidden: torch.Tensor, rotation: torch.Tensor, *, layer: int, plan: DualKVPlan,
|
| 195 |
+
maps: UpMaps | None, video_start: int, dense_rotation: torch.Tensor,
|
| 196 |
+
capture: dict | None = None) -> torch.Tensor:
|
| 197 |
+
"""H3 ``Attention.forward`` with the dense K/V view for 1x1 queries (see module docstring).
|
| 198 |
+
|
| 199 |
+
``hidden`` is the packed ``(1, S, D)`` sequence whose tail from ``video_start`` is the LoT video;
|
| 200 |
+
``rotation`` its rotary table; ``dense_rotation`` the table of the dense video positions.
|
| 201 |
+
``capture`` (optional) receives this layer's pre-RoPE video keys/values for ``UpMaps.fit_``.
|
| 202 |
+
``maps=None`` is H3's own attention (capture only).
|
| 203 |
+
"""
|
| 204 |
+
batch, length, _ = hidden.shape
|
| 205 |
+
query, key, value = attn.qkv_proj(hidden).split(attn.inner_dim, dim=-1)
|
| 206 |
+
query = attn.q_norm(query.reshape(batch, length, attn.num_heads, attn.head_dim))
|
| 207 |
+
key = attn.k_norm(key.reshape(batch, length, attn.num_heads, attn.head_dim))
|
| 208 |
+
value = value.reshape(batch, length, attn.num_heads, attn.head_dim).contiguous()
|
| 209 |
+
if capture is not None:
|
| 210 |
+
capture[layer] = (key[:, video_start:].detach(), value[:, video_start:].detach())
|
| 211 |
+
query_r = _rope(query, rotation).to(value)
|
| 212 |
+
key_r = _rope(key, rotation).to(value)
|
| 213 |
+
if maps is None:
|
| 214 |
+
return attn.out_proj(_sdpa(query_r, key_r, value))
|
| 215 |
+
|
| 216 |
+
# Native LoT attention for text, audio and coarse queries.
|
| 217 |
+
other = torch.cat([torch.arange(video_start, device=hidden.device), video_start + plan.other_rows])
|
| 218 |
+
output = hidden.new_zeros(batch, length, attn.inner_dim, dtype=value.dtype)
|
| 219 |
+
output[:, other] = _sdpa(query_r[:, other], key_r, value)
|
| 220 |
+
|
| 221 |
+
if plan.fine_rows.numel():
|
| 222 |
+
# Dense view for 1x1 queries: fine keys as they are, coarse keys expanded to their cells.
|
| 223 |
+
video_key, video_value = key[:, video_start:], value[:, video_start:]
|
| 224 |
+
keys = [video_key[:, plan.fine_rows]]
|
| 225 |
+
values = [video_value[:, plan.fine_rows]]
|
| 226 |
+
for extent, rows in plan.groups:
|
| 227 |
+
keys.append(maps.expand(layer, extent, "k", video_key[:, rows]))
|
| 228 |
+
values.append(maps.expand(layer, extent, "v", video_value[:, rows]))
|
| 229 |
+
dense_key = torch.cat(keys, dim=1)[:, plan.dense_order]
|
| 230 |
+
dense_value = torch.cat(values, dim=1)[:, plan.dense_order]
|
| 231 |
+
dense_key = _rope(dense_key, dense_rotation).to(value)
|
| 232 |
+
fine = video_start + plan.fine_rows
|
| 233 |
+
output[:, fine] = _sdpa(query_r[:, fine],
|
| 234 |
+
torch.cat([key_r[:, :video_start], dense_key], dim=1),
|
| 235 |
+
torch.cat([value[:, :video_start], dense_value], dim=1))
|
| 236 |
+
return attn.out_proj(output)
|
| 237 |
+
|
| 238 |
+
|
| 239 |
+
@contextlib.contextmanager
|
| 240 |
+
def patched_attention(blocks, attention=None, **context):
|
| 241 |
+
"""Route every block's ``attn.forward`` through ``dual_attention`` for one forward pass.
|
| 242 |
+
|
| 243 |
+
musubi-tuner's code is unchanged: the instance attribute shadows the class method and is
|
| 244 |
+
removed on exit, so checkpointing and block swap see the same modules."""
|
| 245 |
+
patched = []
|
| 246 |
+
try:
|
| 247 |
+
for layer, block in enumerate(blocks):
|
| 248 |
+
attn = block.attn
|
| 249 |
+
|
| 250 |
+
def forward(self, hidden, rotation=None, _layer=layer):
|
| 251 |
+
return (attention or dual_attention)(self, hidden, rotation, layer=_layer, **context)
|
| 252 |
+
|
| 253 |
+
attn.forward = types.MethodType(forward, attn)
|
| 254 |
+
patched.append(attn)
|
| 255 |
+
yield
|
| 256 |
+
finally:
|
| 257 |
+
for attn in patched:
|
| 258 |
+
del attn.forward
|
| 259 |
+
|
| 260 |
+
|
| 261 |
+
# ------------------------------------------------------------- 4. two native tracks
|
| 262 |
+
|
| 263 |
+
|
| 264 |
+
class TwoTrackPlan:
|
| 265 |
+
"""Index tensors for the paper's own arrangement: a complete native LR grid plus HR tokens on a mask.
|
| 266 |
+
|
| 267 |
+
``mask`` is ``(T, h, w)`` bool on the LR token grid; every masked LR token has ``scale * scale``
|
| 268 |
+
HR tokens. ``hr_index`` are their rows in the HR grid's raster order (the order of the HR
|
| 269 |
+
track's rows), and ``cell_order`` permutes "LR token, then its cells row-major" into HR raster.
|
| 270 |
+
"""
|
| 271 |
+
|
| 272 |
+
def __init__(self, mask: torch.Tensor, scale: int = 2):
|
| 273 |
+
mask = mask.bool().cpu()
|
| 274 |
+
time, height, width = mask.shape
|
| 275 |
+
self.mask, self.scale = mask, int(scale)
|
| 276 |
+
self.lr_count = time * height * width
|
| 277 |
+
self.hr_shape = (time, height * scale, width * scale)
|
| 278 |
+
up = mask.repeat_interleave(scale, 1).repeat_interleave(scale, 2)
|
| 279 |
+
self.hr_index = torch.nonzero(up.flatten()).squeeze(1)
|
| 280 |
+
t, u, v = torch.meshgrid(torch.arange(time), torch.arange(height), torch.arange(width), indexing="ij")
|
| 281 |
+
du, dv = torch.meshgrid(torch.arange(scale), torch.arange(scale), indexing="ij")
|
| 282 |
+
cells = (t.reshape(-1, 1) * (height * scale) + u.reshape(-1, 1) * scale + du.reshape(1, -1)) * (width * scale) \
|
| 283 |
+
+ v.reshape(-1, 1) * scale + dv.reshape(1, -1) # (lr, scale^2)
|
| 284 |
+
self.cell_order = torch.argsort(cells.flatten())
|
| 285 |
+
self.extent = (1, self.scale, self.scale)
|
| 286 |
+
|
| 287 |
+
@property
|
| 288 |
+
def hr_count(self) -> int:
|
| 289 |
+
return int(self.hr_index.numel())
|
| 290 |
+
|
| 291 |
+
def grid_source(self, device) -> torch.Tensor:
|
| 292 |
+
"""For each HR-grid cell (raster order), its row in ``cat([LR tokens, HR tokens])``: the LR token that
|
| 293 |
+
covers it, or its own HR token inside the mask. One gather builds the whole grid."""
|
| 294 |
+
cached = getattr(self, "_grid_source", None)
|
| 295 |
+
if cached is None or cached.device != torch.device(device):
|
| 296 |
+
source = torch.div(self.cell_order, self.scale * self.scale, rounding_mode="floor")
|
| 297 |
+
source[self.hr_index] = self.lr_count + torch.arange(self.hr_count, device=source.device)
|
| 298 |
+
self._grid_source = cached = source.to(device)
|
| 299 |
+
return cached
|
| 300 |
+
|
| 301 |
+
def to(self, device) -> "TwoTrackPlan":
|
| 302 |
+
self.hr_index = self.hr_index.to(device)
|
| 303 |
+
self.cell_order = self.cell_order.to(device)
|
| 304 |
+
return self
|
| 305 |
+
|
| 306 |
+
|
| 307 |
+
def two_track_attention(attn, hidden: torch.Tensor, rotation: torch.Tensor, *, layer: int, plan: TwoTrackPlan,
|
| 308 |
+
maps: UpMaps | None, video_start: int, hr_rotation: torch.Tensor,
|
| 309 |
+
capture: dict | None = None) -> torch.Tensor:
|
| 310 |
+
"""Dual-resolution attention (their eq. 3) on ``[text, audio, LR video (whole grid), HR video (mask)]``.
|
| 311 |
+
|
| 312 |
+
Text, audio and LR queries see only text, audio and LR keys: that part is the unmodified model
|
| 313 |
+
at LR, so it needs no ``f_down``. HR queries see text, audio and a full HR grid of keys/values:
|
| 314 |
+
their own inside the mask, ``f_up`` of the LR token's outside it (``maps=None`` copies), rotated
|
| 315 |
+
at the HR grid's positions (``hr_rotation``, the table of the whole HR grid).
|
| 316 |
+
"""
|
| 317 |
+
batch, length, _ = hidden.shape
|
| 318 |
+
query, key, value = attn.qkv_proj(hidden).split(attn.inner_dim, dim=-1)
|
| 319 |
+
query = attn.q_norm(query.reshape(batch, length, attn.num_heads, attn.head_dim))
|
| 320 |
+
key = attn.k_norm(key.reshape(batch, length, attn.num_heads, attn.head_dim))
|
| 321 |
+
value = value.reshape(batch, length, attn.num_heads, attn.head_dim).contiguous()
|
| 322 |
+
split = video_start + plan.lr_count
|
| 323 |
+
if capture is not None:
|
| 324 |
+
capture[layer] = (key[:, video_start:split].detach(), value[:, video_start:split].detach())
|
| 325 |
+
query_r = _rope(query, rotation).to(value)
|
| 326 |
+
key_r = _rope(key[:, :split], rotation[:, :split]).to(value) # HR rows' own rotated keys are never read
|
| 327 |
+
output = hidden.new_zeros(batch, length, attn.inner_dim, dtype=value.dtype)
|
| 328 |
+
output[:, :split] = _sdpa(query_r[:, :split], key_r, value[:, :split])
|
| 329 |
+
if plan.hr_count:
|
| 330 |
+
lr_key, lr_value = key[:, video_start:split], value[:, video_start:split]
|
| 331 |
+
if maps is None:
|
| 332 |
+
# One gather builds the HR grid: each cell takes its LR token's key (copy) or its own HR key.
|
| 333 |
+
source = plan.grid_source(lr_key.device)
|
| 334 |
+
grid_key = torch.cat([lr_key, key[:, split:]], dim=1)[:, source]
|
| 335 |
+
grid_value = torch.cat([lr_value, value[:, split:]], dim=1)[:, source]
|
| 336 |
+
else:
|
| 337 |
+
grid_key = maps.expand(layer, plan.extent, "k", lr_key).to(key.dtype)
|
| 338 |
+
grid_value = maps.expand(layer, plan.extent, "v", lr_value).to(value.dtype)
|
| 339 |
+
grid_key = grid_key[:, plan.cell_order].index_copy(1, plan.hr_index, key[:, split:])
|
| 340 |
+
grid_value = grid_value[:, plan.cell_order].index_copy(1, plan.hr_index, value[:, split:])
|
| 341 |
+
grid_key = _rope(grid_key, hr_rotation).to(value)
|
| 342 |
+
output[:, split:] = _sdpa(query_r[:, split:],
|
| 343 |
+
torch.cat([key_r[:, :video_start], grid_key], dim=1),
|
| 344 |
+
torch.cat([value[:, :video_start], grid_value], dim=1))
|
| 345 |
+
return attn.out_proj(output)
|
| 346 |
+
|
| 347 |
+
|
| 348 |
+
# ------------------------------------------------------------------ 1. region head
|
| 349 |
+
|
| 350 |
+
|
| 351 |
+
class RegionHead(nn.Module):
|
| 352 |
+
"""``s = w^T LN(h)`` per video token, on a middle block's states (their eq. 7)."""
|
| 353 |
+
|
| 354 |
+
def __init__(self, hidden_size: int):
|
| 355 |
+
super().__init__()
|
| 356 |
+
self.norm = nn.LayerNorm(hidden_size, elementwise_affine=False)
|
| 357 |
+
self.score = nn.Linear(hidden_size, 1)
|
| 358 |
+
nn.init.zeros_(self.score.weight)
|
| 359 |
+
nn.init.zeros_(self.score.bias)
|
| 360 |
+
|
| 361 |
+
def forward(self, states: torch.Tensor) -> torch.Tensor:
|
| 362 |
+
"""``(..., L, D)`` -> ``(..., L)`` logits."""
|
| 363 |
+
return self.score(self.norm(states.float())).squeeze(-1)
|
| 364 |
+
|
| 365 |
+
|
| 366 |
+
def scores_to_cells(scores: torch.Tensor, layout: LotLayout) -> torch.Tensor:
|
| 367 |
+
"""Per-token scores ``(L,)`` -> ``(T, H, W)``: every cell takes its token's score."""
|
| 368 |
+
field = scores.new_zeros(layout.time, layout.height, layout.width)
|
| 369 |
+
for value, rect in zip(scores, layout.rects):
|
| 370 |
+
field[rect.t, rect.u:rect.u + rect.eh, rect.v:rect.v + rect.ew] = value
|
| 371 |
+
return field
|
| 372 |
+
|
| 373 |
+
|
| 374 |
+
def region_mask(field: torch.Tensor, *, alpha: float = 0.5, floor: float | None = None,
|
| 375 |
+
sigma: float = 1.0, temporal: bool = True) -> torch.Tensor:
|
| 376 |
+
"""Their selection rule: smooth, then ``M = s~ > m + alpha (p - m)`` per video.
|
| 377 |
+
|
| 378 |
+
``field`` is ``(T, H, W)`` scores (logits or probabilities). Smoothing is a spatial
|
| 379 |
+
Gaussian and, with ``temporal``, a ``[1, 2, 1] / 4`` filter across latent frames. With
|
| 380 |
+
``floor``, a video whose 99th percentile is under it stays fully coarse."""
|
| 381 |
+
smoothed = smooth_detail(field.float(), sigma)
|
| 382 |
+
if temporal and smoothed.shape[0] > 1:
|
| 383 |
+
padded = torch.cat([smoothed[:1], smoothed, smoothed[-1:]])
|
| 384 |
+
smoothed = 0.25 * padded[:-2] + 0.5 * padded[1:-1] + 0.25 * padded[2:]
|
| 385 |
+
flat = smoothed.flatten()
|
| 386 |
+
median, top = float(flat.median()), float(torch.quantile(flat, 0.99))
|
| 387 |
+
if floor is not None and top < floor:
|
| 388 |
+
return torch.zeros_like(smoothed, dtype=torch.bool)
|
| 389 |
+
return smoothed > median + alpha * (top - median)
|
| 390 |
+
|
| 391 |
+
|
| 392 |
+
def region_layout(mask: torch.Tensor, *, background: int = 4, ring: int = 1) -> LotLayout:
|
| 393 |
+
"""Mask -> per-latent-frame LoT layout: 1x1 inside, 2x2 on a ``ring``-token border, ``background`` elsewhere."""
|
| 394 |
+
desired = torch.full(mask.shape, int(background), dtype=torch.long)
|
| 395 |
+
if ring > 0 and background > 2:
|
| 396 |
+
grown = F.max_pool2d(mask.float()[:, None], 2 * ring + 1, stride=1, padding=ring)[:, 0] > 0.5
|
| 397 |
+
desired[grown] = 2
|
| 398 |
+
desired[mask] = 1
|
| 399 |
+
return layout_from_desired_any(desired)
|
| 400 |
+
|
| 401 |
+
|
| 402 |
+
def region_labels(x0_coarse: torch.Tensor, x0_dense: torch.Tensor, fraction: float = 0.25) -> torch.Tensor:
|
| 403 |
+
"""Self-supervised label ``(T, H, W)``: the ``fraction`` of tokens per latent frame where the
|
| 404 |
+
coarse layout's clean estimate is furthest from the dense model's (a latent-space stand-in
|
| 405 |
+
for their per-patch LPIPS label). Latents are ``(1, C, T, 2H, 2W)``."""
|
| 406 |
+
error = (x0_coarse.float() - x0_dense.float()).abs().mean(dim=1)[0] # (T, 2H, 2W)
|
| 407 |
+
error = F.avg_pool2d(error[:, None], 2)[:, 0] # token grid
|
| 408 |
+
flat = error.flatten(1)
|
| 409 |
+
cut = torch.quantile(flat, 1.0 - fraction, dim=1, keepdim=True)
|
| 410 |
+
return (flat >= cut).reshape(error.shape)
|
| 411 |
+
|
| 412 |
+
|
| 413 |
+
def token_targets(labels: torch.Tensor, layout: LotLayout) -> torch.Tensor:
|
| 414 |
+
"""Cell labels ``(T, H, W)`` -> per-token targets ``(L,)``: the labelled share of its cells."""
|
| 415 |
+
return torch.stack([labels[r.t, r.u:r.u + r.eh, r.v:r.v + r.ew].float().mean() for r in layout.rects])
|
| 416 |
+
|
| 417 |
+
|
| 418 |
+
# ------------------------------------------------------------------- 3. seam mixer
|
| 419 |
+
|
| 420 |
+
|
| 421 |
+
def extent_maps(layout: LotLayout) -> torch.Tensor:
|
| 422 |
+
"""``(2, T, H, W)``: is-1x1 mask and ``log2`` token area, on the token grid."""
|
| 423 |
+
fine = torch.zeros(layout.time, layout.height, layout.width)
|
| 424 |
+
area = torch.zeros_like(fine)
|
| 425 |
+
for rect in layout.rects:
|
| 426 |
+
fine[rect.t, rect.u:rect.u + rect.eh, rect.v:rect.v + rect.ew] = float(rect.extent == (1, 1, 1))
|
| 427 |
+
area[rect.t, rect.u:rect.u + rect.eh, rect.v:rect.v + rect.ew] = math.log2(rect.eh * rect.ew)
|
| 428 |
+
return torch.stack([fine, area])
|
| 429 |
+
|
| 430 |
+
|
| 431 |
+
class SeamMixer(nn.Module):
|
| 432 |
+
"""``z = M z_lot + (1 - M)(z_lot + G(z_lot, maps))`` on the latent (their eq. 6, adapted).
|
| 433 |
+
|
| 434 |
+
``G`` is three 3x3x3 convs; the last is zero-initialised, so the mixer starts as the identity.
|
| 435 |
+
``M`` is the 1x1 region upsampled to latent pixels."""
|
| 436 |
+
|
| 437 |
+
def __init__(self, channels: int = 24, width: int = 64):
|
| 438 |
+
super().__init__()
|
| 439 |
+
self.net = nn.Sequential(
|
| 440 |
+
nn.Conv3d(channels + 2, width, 3, padding=1), nn.SiLU(),
|
| 441 |
+
nn.Conv3d(width, width, 3, padding=1), nn.SiLU(),
|
| 442 |
+
nn.Conv3d(width, channels, 3, padding=1),
|
| 443 |
+
)
|
| 444 |
+
nn.init.zeros_(self.net[-1].weight)
|
| 445 |
+
nn.init.zeros_(self.net[-1].bias)
|
| 446 |
+
|
| 447 |
+
def forward(self, latent: torch.Tensor, layout: LotLayout) -> torch.Tensor:
|
| 448 |
+
maps = extent_maps(layout).to(latent.device, latent.dtype)
|
| 449 |
+
maps = maps.repeat_interleave(2, -2).repeat_interleave(2, -1).unsqueeze(0) # (1, 2, T, 2H, 2W)
|
| 450 |
+
residual = self.net(torch.cat([latent, maps], dim=1))
|
| 451 |
+
return latent + (1.0 - maps[:, :1]) * residual
|
| 452 |
+
|
| 453 |
+
|
| 454 |
+
def mixer_loss(mixed: torch.Tensor, target: torch.Tensor, layout: LotLayout) -> torch.Tensor:
|
| 455 |
+
"""Latent L1 outside the 1x1 region (their latent term; pixel L1 + LPIPS need the VAE decoder)."""
|
| 456 |
+
outside = 1.0 - extent_maps(layout)[0].to(mixed.device)
|
| 457 |
+
outside = outside.repeat_interleave(2, -2).repeat_interleave(2, -1)[None, None]
|
| 458 |
+
return ((mixed - target).abs() * outside).sum() / (outside.sum() * mixed.shape[1]).clamp(min=1.0)
|
| 459 |
+
|
| 460 |
+
|
| 461 |
+
# ------------------------------------------------- 5. two-track decoder (their Sec. 4.2)
|
| 462 |
+
|
| 463 |
+
|
| 464 |
+
class LatentUpscaler(nn.Module):
|
| 465 |
+
"""``U``: LR latent ``(B, C, T, h, w)`` -> ``(B, C, T, 2h, 2w)``, bicubic plus a learned residual.
|
| 466 |
+
|
| 467 |
+
The last conv is zero-initialised, so an untrained upscaler is bicubic."""
|
| 468 |
+
|
| 469 |
+
def __init__(self, channels: int = 24, width: int = 128):
|
| 470 |
+
super().__init__()
|
| 471 |
+
self.net = nn.Sequential(
|
| 472 |
+
nn.Conv2d(channels, width, 3, padding=1), nn.SiLU(),
|
| 473 |
+
nn.Conv2d(width, width, 3, padding=1), nn.SiLU(),
|
| 474 |
+
nn.Conv2d(width, 4 * width, 3, padding=1), nn.PixelShuffle(2), nn.SiLU(),
|
| 475 |
+
nn.Conv2d(width, width, 3, padding=1), nn.SiLU(),
|
| 476 |
+
nn.Conv2d(width, channels, 3, padding=1),
|
| 477 |
+
)
|
| 478 |
+
nn.init.zeros_(self.net[-1].weight)
|
| 479 |
+
nn.init.zeros_(self.net[-1].bias)
|
| 480 |
+
|
| 481 |
+
def forward(self, latent: torch.Tensor) -> torch.Tensor:
|
| 482 |
+
batch, channels, time, height, width = latent.shape
|
| 483 |
+
frames = latent.permute(0, 2, 1, 3, 4).reshape(batch * time, channels, height, width).float()
|
| 484 |
+
up = F.interpolate(frames, scale_factor=2, mode="bicubic", align_corners=False) + self.net(frames)
|
| 485 |
+
return up.reshape(batch, time, channels, 2 * height, 2 * width).permute(0, 2, 1, 3, 4)
|
| 486 |
+
|
| 487 |
+
|
| 488 |
+
class TwoTrackMixer(nn.Module):
|
| 489 |
+
"""``z = M z_h + (1 - M)(z_l_up + G(z_h, z_l_up, M))`` (their eq. 6). ``G`` is a small 3D-conv net,
|
| 490 |
+
zero-initialised, so an untrained mixer is a hard paste."""
|
| 491 |
+
|
| 492 |
+
def __init__(self, channels: int = 24, width: int = 64):
|
| 493 |
+
super().__init__()
|
| 494 |
+
self.net = nn.Sequential(
|
| 495 |
+
nn.Conv3d(2 * channels + 1, width, 3, padding=1), nn.SiLU(),
|
| 496 |
+
nn.Conv3d(width, width, 3, padding=1), nn.SiLU(),
|
| 497 |
+
nn.Conv3d(width, channels, 3, padding=1),
|
| 498 |
+
)
|
| 499 |
+
nn.init.zeros_(self.net[-1].weight)
|
| 500 |
+
nn.init.zeros_(self.net[-1].bias)
|
| 501 |
+
|
| 502 |
+
def forward(self, hr: torch.Tensor, lr_up: torch.Tensor, mask: torch.Tensor) -> torch.Tensor:
|
| 503 |
+
"""``hr``, ``lr_up`` ``(B, C, T, H, W)``; ``mask`` ``(B, 1, T, H, W)`` in [0, 1] on the HR latent grid."""
|
| 504 |
+
hr, lr_up, mask = hr.float(), lr_up.float(), mask.float()
|
| 505 |
+
delta = self.net(torch.cat([hr * mask, lr_up, mask], dim=1))
|
| 506 |
+
return mask * hr + (1.0 - mask) * (lr_up + delta)
|
| 507 |
+
|
| 508 |
+
|
| 509 |
+
__all__ = [
|
| 510 |
+
"DualKVPlan", "UpMaps", "DualKV", "dual_attention", "patched_attention", "TwoTrackPlan", "two_track_attention", "RegionHead", "scores_to_cells",
|
| 511 |
+
"region_mask", "region_layout", "region_labels", "token_targets", "SeamMixer", "extent_maps",
|
| 512 |
+
"mixer_loss", "LatentUpscaler", "TwoTrackMixer",
|
| 513 |
+
]
|
lot/procrustes_h3.py
ADDED
|
@@ -0,0 +1,223 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Phase 4: Procrustes-fit the H3 extent bank, then optionally one frozen smoke.
|
| 3 |
+
|
| 4 |
+
python3 lot/procrustes_h3.py --pairs DIR [--smoke] [--out FILE]
|
| 5 |
+
|
| 6 |
+
``DIR`` holds one ``{et}x{eh}x{ew}.pt`` per extent, each a dict with
|
| 7 |
+
``dense (N, 96*et*eh*ew)`` and ``reference (N, 96)`` in ``gather_extent`` row
|
| 8 |
+
order. The user supplies these. This script never builds pairs, and exits if
|
| 9 |
+
the directory is missing. There is no ``1x1x1.pt``: a unit extent is the
|
| 10 |
+
identity and must stay so.
|
| 11 |
+
|
| 12 |
+
The fit reads only ``video_patch_proj`` and ``final_layer.video_out`` from the
|
| 13 |
+
checkpoint, on CPU. ``--smoke`` loads the int8 DiT (32 blocks streamed) and runs one
|
| 14 |
+
frozen forward on the 384×640, ``latent_t=7`` canvas in y-space. It asserts
|
| 15 |
+
shape, finite values, and the packed length. It does not judge picture
|
| 16 |
+
quality: a frozen base sees fitted scales it was never trained on.
|
| 17 |
+
|
| 18 |
+
Nothing is written unless ``--out`` is given. Does not call empty_cache.
|
| 19 |
+
"""
|
| 20 |
+
|
| 21 |
+
from __future__ import annotations
|
| 22 |
+
|
| 23 |
+
import argparse
|
| 24 |
+
import math
|
| 25 |
+
import sys
|
| 26 |
+
from pathlib import Path
|
| 27 |
+
|
| 28 |
+
import torch
|
| 29 |
+
|
| 30 |
+
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
| 31 |
+
from lot_paths import LOT_H3_CHECKPOINT, LOT_H3_STILLS_CACHE, LOT_H3_STILLS_CAPTIONS, LOT_H3_TEXT_ENCODER, LOT_H3_VAE # noqa: E402,F401
|
| 32 |
+
|
| 33 |
+
from adapter import LotVisualAdapter, _key # noqa: E402
|
| 34 |
+
from h3 import H3_EXTENTS, H3_TOKEN_DIM, clip_layout, make_h3_adapter # noqa: E402
|
| 35 |
+
|
| 36 |
+
CHECKPOINT = LOT_H3_CHECKPOINT
|
| 37 |
+
UNIT = (1, 1, 1)
|
| 38 |
+
ORTHO_TOL = 1e-4
|
| 39 |
+
|
| 40 |
+
# Phase-5 canvas: 384×640 -> latent (24, 40), tokens 12×20, 7 latent frames.
|
| 41 |
+
SMOKE_T, SMOKE_H, SMOKE_W, SMOKE_TEXT = 7, 24, 40, 128
|
| 42 |
+
SMOKE_SIGMA = 0.4
|
| 43 |
+
SMOKE_SWAP = 32
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def parse_extent(stem: str) -> tuple[int, int, int]:
|
| 47 |
+
parts = stem.split("x")
|
| 48 |
+
if len(parts) != 3 or not all(part.isdigit() for part in parts):
|
| 49 |
+
raise ValueError(f"pair file {stem!r} is not named {{et}}x{{eh}}x{{ew}}")
|
| 50 |
+
return tuple(int(part) for part in parts)
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def load_pairs(directory: Path) -> dict[tuple[int, int, int], tuple[torch.Tensor, torch.Tensor]]:
|
| 54 |
+
"""Read and validate every pair file. Exits if the directory is missing or empty."""
|
| 55 |
+
if not directory.is_dir():
|
| 56 |
+
raise SystemExit(f"pair directory {directory} does not exist. Phase 4 does not invent pairs.")
|
| 57 |
+
files = sorted(directory.glob("*.pt"))
|
| 58 |
+
if not files:
|
| 59 |
+
raise SystemExit(f"pair directory {directory} has no *.pt files.")
|
| 60 |
+
pairs = {}
|
| 61 |
+
for path in files:
|
| 62 |
+
extent = parse_extent(path.stem)
|
| 63 |
+
if extent == UNIT:
|
| 64 |
+
raise ValueError("1x1x1.pt: a unit extent is the identity lift and is not fit")
|
| 65 |
+
if extent not in H3_EXTENTS:
|
| 66 |
+
raise ValueError(f"{path.name}: extent {extent} is not an H3 extent")
|
| 67 |
+
blob = torch.load(path, map_location="cpu", weights_only=True)
|
| 68 |
+
if not isinstance(blob, dict) or "dense" not in blob or "reference" not in blob:
|
| 69 |
+
raise ValueError(f"{path.name}: expected a dict with 'dense' and 'reference'")
|
| 70 |
+
pairs[extent] = check_pair(extent, blob["dense"], blob["reference"], path.name)
|
| 71 |
+
return pairs
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def check_pair(extent, dense: torch.Tensor, reference: torch.Tensor, name: str = "pair"):
|
| 75 |
+
et, eh, ew = extent
|
| 76 |
+
dense_dim = H3_TOKEN_DIM * et * eh * ew
|
| 77 |
+
if dense.ndim != 2 or dense.shape[1] != dense_dim:
|
| 78 |
+
raise ValueError(f"{name}: dense {tuple(dense.shape)} != (N, {dense_dim})")
|
| 79 |
+
if reference.ndim != 2 or reference.shape[1] != H3_TOKEN_DIM:
|
| 80 |
+
raise ValueError(f"{name}: reference {tuple(reference.shape)} != (N, {H3_TOKEN_DIM})")
|
| 81 |
+
if dense.shape[0] != reference.shape[0]:
|
| 82 |
+
raise ValueError(f"{name}: {dense.shape[0]} dense rows vs {reference.shape[0]} reference rows")
|
| 83 |
+
if dense.shape[0] < H3_TOKEN_DIM:
|
| 84 |
+
raise ValueError(f"{name}: {dense.shape[0]} rows cannot fix a rank-{H3_TOKEN_DIM} basis")
|
| 85 |
+
if not (torch.isfinite(dense).all() and torch.isfinite(reference).all()):
|
| 86 |
+
raise ValueError(f"{name}: non-finite values")
|
| 87 |
+
return dense.float(), reference.float()
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
def pretrained_maps(checkpoint: Path) -> tuple[torch.Tensor, ...]:
|
| 91 |
+
"""``(W_in, W_out, b_in, b_out)`` read lazily from the checkpoint, fp32, CPU."""
|
| 92 |
+
from safetensors import safe_open
|
| 93 |
+
|
| 94 |
+
with safe_open(str(checkpoint), "pt") as handle:
|
| 95 |
+
return tuple(
|
| 96 |
+
handle.get_tensor(key).float()
|
| 97 |
+
for key in (
|
| 98 |
+
"video_patch_proj.weight",
|
| 99 |
+
"final_layer.video_out.weight",
|
| 100 |
+
"video_patch_proj.bias",
|
| 101 |
+
"final_layer.video_out.bias",
|
| 102 |
+
)
|
| 103 |
+
)
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
def fit_bank(adapter: LotVisualAdapter, pairs) -> list[dict]:
|
| 107 |
+
"""``fit_extent`` per pair, then the plan's three checks. Raises on any failure."""
|
| 108 |
+
if int(adapter.pretrained_ready) != 1:
|
| 109 |
+
raise RuntimeError("call init_from_pretrained first; fit_extent rebuilds heads from it")
|
| 110 |
+
unit = _key(UNIT)
|
| 111 |
+
before = {
|
| 112 |
+
name: tensor.detach().clone()
|
| 113 |
+
for name, tensor in (
|
| 114 |
+
*adapter.in_proj[unit].state_dict().items(),
|
| 115 |
+
*(("out_" + k, v) for k, v in adapter.out_proj[unit].state_dict().items()),
|
| 116 |
+
)
|
| 117 |
+
}
|
| 118 |
+
reports = []
|
| 119 |
+
for extent in sorted(pairs):
|
| 120 |
+
dense, reference = pairs[extent]
|
| 121 |
+
adapter.fit_extent(extent, dense, reference)
|
| 122 |
+
basis = adapter.bank.basis(extent).double()
|
| 123 |
+
eye = torch.eye(basis.shape[1], dtype=torch.float64)
|
| 124 |
+
ortho_err = float((basis.T @ basis - eye).abs().max())
|
| 125 |
+
scale = float(adapter.bank.scale(extent))
|
| 126 |
+
if ortho_err > ORTHO_TOL:
|
| 127 |
+
raise ValueError(f"{extent}: A^T A differs from I by {ortho_err:.3g}")
|
| 128 |
+
if not math.isfinite(scale) or scale <= 0:
|
| 129 |
+
raise ValueError(f"{extent}: scale {scale} is not finite and positive")
|
| 130 |
+
reports.append({"extent": extent, "rows": dense.shape[0], "scale": scale, "ortho_err": ortho_err})
|
| 131 |
+
after = {
|
| 132 |
+
name: tensor
|
| 133 |
+
for name, tensor in (
|
| 134 |
+
*adapter.in_proj[unit].state_dict().items(),
|
| 135 |
+
*(("out_" + k, v) for k, v in adapter.out_proj[unit].state_dict().items()),
|
| 136 |
+
)
|
| 137 |
+
}
|
| 138 |
+
for name, tensor in before.items():
|
| 139 |
+
if not torch.equal(tensor, after[name]):
|
| 140 |
+
raise ValueError(f"1x1x1 head {name} changed during the fit")
|
| 141 |
+
if float(adapter.bank.scale(UNIT)) != 1.0:
|
| 142 |
+
raise ValueError("1x1x1 scale moved off 1")
|
| 143 |
+
return reports
|
| 144 |
+
|
| 145 |
+
|
| 146 |
+
def smoke(adapter: LotVisualAdapter) -> None:
|
| 147 |
+
"""One frozen y-space forward on the mixed phase-5 canvas."""
|
| 148 |
+
from flow import sample_noisy
|
| 149 |
+
from gpu_guard import refuse_if_busy
|
| 150 |
+
from h3_splice import LotSplice
|
| 151 |
+
from h3_musubi import audio_frames, load_dit, predict
|
| 152 |
+
|
| 153 |
+
refuse_if_busy("procrustes_h3.py")
|
| 154 |
+
model = load_dit(CHECKPOINT, swap=SMOKE_SWAP)
|
| 155 |
+
|
| 156 |
+
layout = clip_layout(SMOKE_T, SMOKE_H // 2, SMOKE_W // 2)
|
| 157 |
+
adapter.cuda()
|
| 158 |
+
splice = LotSplice(adapter, layout, y_space=True)
|
| 159 |
+
# A random stand-in clean latent. It is a smoke input, not a pair.
|
| 160 |
+
x0 = torch.randn(1, 24, SMOKE_T, SMOKE_H, SMOKE_W, device="cuda", dtype=torch.float32)
|
| 161 |
+
y0 = splice.to_y(x0)
|
| 162 |
+
y_t, _noise = sample_noisy(y0, SMOKE_SIGMA)
|
| 163 |
+
text = torch.randn(1, SMOKE_TEXT, model.config.text_dim, device="cuda", dtype=torch.bfloat16)
|
| 164 |
+
n_audio = audio_frames(SMOKE_T)
|
| 165 |
+
seen = []
|
| 166 |
+
hook = model.blocks[0].register_forward_hook(lambda _m, inputs, _o: seen.append(tuple(inputs[0].shape[1:])))
|
| 167 |
+
try:
|
| 168 |
+
with torch.inference_mode():
|
| 169 |
+
out = predict(model, y_t, 1.0 - SMOKE_SIGMA, text, lot=splice)
|
| 170 |
+
finally:
|
| 171 |
+
hook.remove()
|
| 172 |
+
expected_seq = SMOKE_TEXT + n_audio * 2 + layout.count
|
| 173 |
+
# H3's head is x0 - eps, so one step to sigma 0 is y_t + sigma * out.
|
| 174 |
+
y0_hat = y_t + SMOKE_SIGMA * out.float()
|
| 175 |
+
x0_hat = splice.to_x(y0_hat)
|
| 176 |
+
ok = (
|
| 177 |
+
tuple(out.shape) == tuple(y_t.shape)
|
| 178 |
+
and bool(torch.isfinite(out).all())
|
| 179 |
+
and bool(torch.isfinite(x0_hat).all())
|
| 180 |
+
and bool(seen)
|
| 181 |
+
and seen[0][0] == expected_seq
|
| 182 |
+
)
|
| 183 |
+
peak_mb = torch.cuda.max_memory_allocated() / (1024 * 1024)
|
| 184 |
+
print(
|
| 185 |
+
"LOT_H3 kind=smoke task=t2va "
|
| 186 |
+
f"latent_t={SMOKE_T} canvas=384x640 lot_tokens={expected_seq} "
|
| 187 |
+
f"seq={seen[0] if seen else None} out={tuple(out.shape)} "
|
| 188 |
+
f"out_rms={float(out.float().pow(2).mean().sqrt()):.4g} "
|
| 189 |
+
f"peak_mb={peak_mb:.1f} dtype=bf16 ok={int(ok)}",
|
| 190 |
+
flush=True,
|
| 191 |
+
)
|
| 192 |
+
if not ok:
|
| 193 |
+
raise SystemExit(1)
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
def main() -> None:
|
| 197 |
+
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
| 198 |
+
parser.add_argument("--pairs", type=Path, required=True, help="directory of {et}x{eh}x{ew}.pt pair files")
|
| 199 |
+
parser.add_argument("--smoke", action="store_true", help="also run one frozen forward on the GPU")
|
| 200 |
+
parser.add_argument("--out", type=Path, default=None, help="save the fitted adapter state here")
|
| 201 |
+
args = parser.parse_args()
|
| 202 |
+
|
| 203 |
+
pairs = load_pairs(args.pairs)
|
| 204 |
+
if not CHECKPOINT.is_file():
|
| 205 |
+
raise SystemExit(f"missing checkpoint {CHECKPOINT}")
|
| 206 |
+
adapter = make_h3_adapter()
|
| 207 |
+
adapter.init_from_pretrained(*pretrained_maps(CHECKPOINT))
|
| 208 |
+
for report in fit_bank(adapter, pairs):
|
| 209 |
+
et, eh, ew = report["extent"]
|
| 210 |
+
print(
|
| 211 |
+
f"LOT_H3 kind=procrustes extent={et}x{eh}x{ew} rows={report['rows']} "
|
| 212 |
+
f"scale={report['scale']:.6g} ortho_err={report['ortho_err']:.3g} ok=1",
|
| 213 |
+
flush=True,
|
| 214 |
+
)
|
| 215 |
+
if args.smoke:
|
| 216 |
+
smoke(adapter)
|
| 217 |
+
if args.out is not None:
|
| 218 |
+
torch.save(adapter.state_dict(), args.out)
|
| 219 |
+
print(f"LOT_H3 kind=saved path={args.out}", flush=True)
|
| 220 |
+
|
| 221 |
+
|
| 222 |
+
if __name__ == "__main__":
|
| 223 |
+
main()
|
lot/two_track_h3.py
ADDED
|
@@ -0,0 +1,946 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""MPEG-Diffusion's own arrangement on H3: a native LR track plus HR tokens on a selected region.
|
| 3 |
+
|
| 4 |
+
python3 lot/two_track_h3.py render --out DIR [--steps 20 --switch 18 --hr-steps 6] [--maps DIR/up_maps_tt.pt]
|
| 5 |
+
[--region DIR/region_head_tt.pt | --box 0.25 0.2 0.75 0.9]
|
| 6 |
+
python3 lot/two_track_h3.py fit-kv --out DIR [--items 24] # ridge fit of f_up, base model at LR and HR
|
| 7 |
+
python3 lot/two_track_h3.py region --out DIR [--items 200] # linear region head on the LR track's states
|
| 8 |
+
|
| 9 |
+
Paper: Dennis Menn, Tianpei Gu, Guande He, Aryan V S, Yanke Song, Nan Liu, Desai Xie, Xun Huang, Chenfeng Xu and Diana Marculescu, MPEG-Diffusion: Mixed-Resolution Video Generation, arXiv:2611.12069.
|
| 10 |
+
|
| 11 |
+
``mixed_res_h3.py`` puts the paper's pieces on LoT's merged tokens, so its coarse region is only as
|
| 12 |
+
good as the LoT adapter. Here the LR track is what the paper uses: the unmodified base model at
|
| 13 |
+
half resolution (no adapter, no LoRA), which is good from the start. H3's rotary positions are
|
| 14 |
+
normalised by ``sqrt(area)``, so the LR and HR grids already share one coordinate frame.
|
| 15 |
+
|
| 16 |
+
Sampler (their Sec. 4, "Inference"):
|
| 17 |
+
|
| 18 |
+
1. ``--switch`` steps denoise the whole frame at LR (``W/2 x H/2``).
|
| 19 |
+
2. At the switch a region ``M`` is chosen on the LR token grid (``--region`` head, or ``--box``).
|
| 20 |
+
3. The clean LR estimate is upscaled, re-noised with fresh noise, and the remaining steps run one
|
| 21 |
+
packed sequence ``[text, audio, LR video (whole grid), HR video (M only)]`` with dual-resolution
|
| 22 |
+
attention (``mixed_res.two_track_attention``): LR queries see only LR keys, so the LR track stays
|
| 23 |
+
the native model; HR queries see their own keys inside ``M`` and ``f_up`` of the LR keys outside.
|
| 24 |
+
4. The LR latent is upscaled and the HR latent is pasted over ``M``; the VAE decodes that.
|
| 25 |
+
|
| 26 |
+
The upscaler ``U`` is the VAE itself (decode, bicubic x2, encode): it needs no training and returns
|
| 27 |
+
a latent the decoder already agrees with. ``--feather`` blends the paste edge in latent space.
|
| 28 |
+
Not trained here (the paper fine-tunes them): the DiT with the maps, the resolution embedding,
|
| 29 |
+
``f_down`` (unused: the LR track is complete), the learned upscaler and the mixer.
|
| 30 |
+
"""
|
| 31 |
+
|
| 32 |
+
from __future__ import annotations
|
| 33 |
+
|
| 34 |
+
import argparse
|
| 35 |
+
import contextlib
|
| 36 |
+
import json
|
| 37 |
+
import os
|
| 38 |
+
import random
|
| 39 |
+
import sys
|
| 40 |
+
import time
|
| 41 |
+
from dataclasses import replace
|
| 42 |
+
from pathlib import Path
|
| 43 |
+
|
| 44 |
+
import torch
|
| 45 |
+
import torch.nn.functional as F
|
| 46 |
+
|
| 47 |
+
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
| 48 |
+
from gpu_guard import refuse_if_busy # noqa: E402
|
| 49 |
+
from mixed_res import (RegionHead, TwoTrackPlan, UpMaps, patched_attention, region_mask, # noqa: E402
|
| 50 |
+
two_track_attention)
|
| 51 |
+
from procrustes_h3 import CHECKPOINT # noqa: E402
|
| 52 |
+
|
| 53 |
+
HEAD_DIM = 128
|
| 54 |
+
MID_BLOCK = 25
|
| 55 |
+
EXTENT = (1, 2, 2)
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
# ------------------------------------------------------------------------- forward
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def two_track_forward(dit, lr: torch.Tensor, hr: torch.Tensor | None, plan: TwoTrackPlan | None, t: float,
|
| 62 |
+
text: torch.Tensor, *, maps: UpMaps | None = None, audio_generator=None,
|
| 63 |
+
capture: dict | None = None, res_embed: "ResolutionEmbedding | None" = None):
|
| 64 |
+
"""Velocities ``(v_lr, v_hr)`` of the LR latent and of the HR latent on the plan's region.
|
| 65 |
+
|
| 66 |
+
``lr`` is ``(1, 24, T, h, w)``, ``hr`` is ``(1, 24, T, 2h, 2w)``; ``v_hr`` is zero outside the
|
| 67 |
+
region. With ``plan=None`` (or an empty region) this is the base model on ``lr`` alone.
|
| 68 |
+
``capture={"block": i}`` returns that block's LR video states in ``capture["states"]``;
|
| 69 |
+
``capture={"kv": {}}`` collects every block's pre-RoPE LR keys and values.
|
| 70 |
+
``res_embed`` adds the paper's learned LR / HR vectors to the video rows.
|
| 71 |
+
|
| 72 |
+
With grad enabled (training) the attention patch is installed for good (``_training_patch``)
|
| 73 |
+
so activation checkpointing recomputes the same attention in the backward pass.
|
| 74 |
+
"""
|
| 75 |
+
from h3_musubi import _layout, _silence, _sync_swap, audio_sigma
|
| 76 |
+
from musubi_tuner.minimax_h3.packing import (H3RowSegment, _unpack_video_rows, build_position_grid,
|
| 77 |
+
build_timestep_rows, pack_audio_rows, pack_video_rows)
|
| 78 |
+
|
| 79 |
+
if text.ndim == 2:
|
| 80 |
+
text = text.unsqueeze(0)
|
| 81 |
+
_sync_swap(dit)
|
| 82 |
+
device = dit.video_patch_proj.weight.device
|
| 83 |
+
sigma_a = float(audio_sigma(1.0 - t))
|
| 84 |
+
_b, _c, frames, lat_h, lat_w = lr.shape
|
| 85 |
+
layout = _layout(text.shape[1], frames, lat_h, lat_w)
|
| 86 |
+
segment = layout.target_video_segment
|
| 87 |
+
if segment.stop != layout.row_count:
|
| 88 |
+
raise RuntimeError("the target video must be the last segment of the packed sequence")
|
| 89 |
+
lr_count = segment.row_count
|
| 90 |
+
hr_count = plan.hr_count if plan is not None else 0
|
| 91 |
+
if plan is not None and plan.lr_count != lr_count:
|
| 92 |
+
raise ValueError(f"plan has {plan.lr_count} LR tokens, the latent has {lr_count}")
|
| 93 |
+
|
| 94 |
+
rows = pack_video_rows(lr.to(device, torch.float32))
|
| 95 |
+
positions = build_position_grid(layout)[0]
|
| 96 |
+
hr_rotation = None
|
| 97 |
+
if hr_count:
|
| 98 |
+
hr_layout = _layout(text.shape[1], frames, 2 * lat_h, 2 * lat_w)
|
| 99 |
+
hr_positions = build_position_grid(hr_layout)[0][-4 * lr_count:]
|
| 100 |
+
rows = torch.cat([rows, pack_video_rows(hr.to(device, torch.float32))[:, plan.hr_index]], dim=1)
|
| 101 |
+
positions = torch.cat([positions, hr_positions[plan.hr_index.cpu()]])
|
| 102 |
+
video = dit.video_patch_proj(rows).to(dit.dtype)
|
| 103 |
+
if res_embed is not None:
|
| 104 |
+
video = res_embed(video, lr_count)
|
| 105 |
+
audio = _silence(frames, sigma_a, device, audio_generator)
|
| 106 |
+
audio_rows = dit.audio_patch_proj(pack_audio_rows(audio.to(device)).to(torch.float32)).to(dit.dtype)
|
| 107 |
+
text_rows = dit.token_refiner(dit.condition_proj(text.to(device, dit.dtype)))
|
| 108 |
+
hidden = torch.cat([text_rows, audio_rows, video], dim=1)
|
| 109 |
+
|
| 110 |
+
segments = layout.segments[:-1] + (H3RowSegment(segment.role, segment.kind, segment.start,
|
| 111 |
+
segment.start + lr_count + hr_count),)
|
| 112 |
+
packed = replace(layout, segments=segments, row_count=segment.start + lr_count + hr_count)
|
| 113 |
+
rotation = dit._rotation_table(positions.unsqueeze(0).to(device), hidden.dtype)
|
| 114 |
+
tags = torch.ones(1, text.shape[1], dtype=torch.int64)
|
| 115 |
+
timesteps = build_timestep_rows(packed, text_token_tags=tags, model_t_video=t, model_t_audio=1.0 - sigma_a)
|
| 116 |
+
temb = dit._timestep_embeddings(timesteps.unique_timesteps, device)
|
| 117 |
+
|
| 118 |
+
patch = contextlib.nullcontext()
|
| 119 |
+
want_kv = capture is not None and "kv" in capture
|
| 120 |
+
training = torch.is_grad_enabled()
|
| 121 |
+
persistent = training or getattr(dit, "_two_track_context", None) is not None
|
| 122 |
+
if hr_count or want_kv or persistent:
|
| 123 |
+
if hr_count:
|
| 124 |
+
hr_rotation = dit._rotation_table(hr_positions.unsqueeze(0).to(device), hidden.dtype)
|
| 125 |
+
context = dict(plan=plan if plan is not None and hr_count else _empty_plan(frames, lat_h, lat_w, device),
|
| 126 |
+
maps=maps, video_start=segment.start, hr_rotation=hr_rotation,
|
| 127 |
+
capture=capture.get("kv") if want_kv else None)
|
| 128 |
+
if persistent:
|
| 129 |
+
_training_patch(dit).update(context)
|
| 130 |
+
else:
|
| 131 |
+
patch = patched_attention(dit.blocks, attention=two_track_attention, **context)
|
| 132 |
+
checkpoint = training and dit.gradient_checkpointing
|
| 133 |
+
with patch:
|
| 134 |
+
for index, block in enumerate(dit.blocks):
|
| 135 |
+
if dit.blocks_to_swap:
|
| 136 |
+
dit.offloader.wait_for_block(index)
|
| 137 |
+
if checkpoint:
|
| 138 |
+
hidden = torch.utils.checkpoint.checkpoint(block, hidden, temb, timesteps.block_segments, rotation,
|
| 139 |
+
use_reentrant=False)
|
| 140 |
+
else:
|
| 141 |
+
hidden = block(hidden, temb, timesteps.block_segments, rotation)
|
| 142 |
+
if dit.blocks_to_swap:
|
| 143 |
+
dit.offloader.submit_move_blocks_forward(dit.blocks, index)
|
| 144 |
+
if capture is not None and capture.get("block") == index:
|
| 145 |
+
states = hidden[0, segment.start:segment.start + lr_count].float()
|
| 146 |
+
capture["states"] = states if capture.get("grad") else states.detach()
|
| 147 |
+
hidden = hidden.to(device)
|
| 148 |
+
|
| 149 |
+
final = dit.final_layer
|
| 150 |
+
shift, scale = final.adaln_proj(temb)
|
| 151 |
+
index = timesteps.video_timestep_index
|
| 152 |
+
states = final.norm(hidden[:, segment.start:]) * (1.0 + scale[index]) + shift[index]
|
| 153 |
+
out = final.video_out(states.to(torch.float32))
|
| 154 |
+
v_lr = _unpack_video_rows(out[:, :lr_count], layout.target_video).to(lr.dtype)
|
| 155 |
+
if not hr_count:
|
| 156 |
+
return v_lr, None
|
| 157 |
+
full = out.new_zeros(1, 4 * lr_count, out.shape[-1])
|
| 158 |
+
full[:, plan.hr_index] = out[:, lr_count:]
|
| 159 |
+
return v_lr, _unpack_video_rows(full, hr_layout.target_video).to(hr.dtype)
|
| 160 |
+
|
| 161 |
+
|
| 162 |
+
def _empty_plan(frames: int, lat_h: int, lat_w: int, device) -> TwoTrackPlan:
|
| 163 |
+
return TwoTrackPlan(torch.zeros(frames, lat_h // 2, lat_w // 2, dtype=torch.bool)).to(device)
|
| 164 |
+
|
| 165 |
+
|
| 166 |
+
def _training_patch(dit) -> dict:
|
| 167 |
+
"""Route every block's attention through ``two_track_attention`` for the life of the process and
|
| 168 |
+
return the shared context dict. A context manager would be gone before the backward pass, and
|
| 169 |
+
checkpointed blocks recompute attention there."""
|
| 170 |
+
context = getattr(dit, "_two_track_context", None)
|
| 171 |
+
if context is None:
|
| 172 |
+
import types
|
| 173 |
+
|
| 174 |
+
context = {}
|
| 175 |
+
for layer, block in enumerate(dit.blocks):
|
| 176 |
+
def forward(self, hidden, rotation=None, _layer=layer):
|
| 177 |
+
return two_track_attention(self, hidden, rotation, layer=_layer, **context)
|
| 178 |
+
|
| 179 |
+
block.attn.forward = types.MethodType(forward, block.attn)
|
| 180 |
+
dit._two_track_context = context
|
| 181 |
+
context.clear()
|
| 182 |
+
return context
|
| 183 |
+
|
| 184 |
+
|
| 185 |
+
class ResolutionEmbedding(torch.nn.Module):
|
| 186 |
+
"""The paper's resolution embedding: one learned vector for LR rows and one for HR rows, zero at init."""
|
| 187 |
+
|
| 188 |
+
def __init__(self, hidden_size: int):
|
| 189 |
+
super().__init__()
|
| 190 |
+
self.lr = torch.nn.Parameter(torch.zeros(hidden_size))
|
| 191 |
+
self.hr = torch.nn.Parameter(torch.zeros(hidden_size))
|
| 192 |
+
|
| 193 |
+
def forward(self, video: torch.Tensor, lr_count: int) -> torch.Tensor:
|
| 194 |
+
add = torch.cat([self.lr.expand(lr_count, -1), self.hr.expand(video.shape[1] - lr_count, -1)])
|
| 195 |
+
return video + add.to(video.dtype).unsqueeze(0)
|
| 196 |
+
|
| 197 |
+
|
| 198 |
+
# ------------------------------------------------------------------------ upscaler
|
| 199 |
+
|
| 200 |
+
|
| 201 |
+
class VaeUpscaler:
|
| 202 |
+
"""``U``: decode, bicubic x2 per frame, encode. A still or a clip (any latent frame count)."""
|
| 203 |
+
|
| 204 |
+
def __init__(self, vae):
|
| 205 |
+
self.vae = vae
|
| 206 |
+
self.dtype = next(vae.parameters()).dtype
|
| 207 |
+
|
| 208 |
+
@torch.no_grad()
|
| 209 |
+
def _resample(self, latent: torch.Tensor, **interp) -> torch.Tensor:
|
| 210 |
+
# Long clips (16 s): the decoded clip leaves the GPU at once as fp16, the resize runs 16 frames at a time,
|
| 211 |
+
# and only the fp16 result goes back for the encode, so the GPU never holds a float32 copy of the clip.
|
| 212 |
+
pixels = self.vae.decode(latent.to("cuda", self.dtype)).clamp(-1, 1).to("cpu", torch.float16)
|
| 213 |
+
if latent.shape[2] == 1:
|
| 214 |
+
pixels = pixels[:, :, :1]
|
| 215 |
+
b, c, f, h, w = pixels.shape
|
| 216 |
+
resized = None
|
| 217 |
+
for start in range(0, f, 16):
|
| 218 |
+
chunk = pixels[0, :, start:start + 16].transpose(0, 1).to("cuda", torch.float32) # (n, 3, h, w)
|
| 219 |
+
chunk = F.interpolate(chunk, **interp).clamp(-1, 1).to("cpu", torch.float16)
|
| 220 |
+
if resized is None:
|
| 221 |
+
resized = torch.empty(1, c, f, *chunk.shape[-2:], dtype=torch.float16)
|
| 222 |
+
resized[0, :, start:start + chunk.shape[0]] = chunk.transpose(0, 1)
|
| 223 |
+
del pixels
|
| 224 |
+
torch.cuda.empty_cache()
|
| 225 |
+
pixels = resized.to("cuda", self.dtype)
|
| 226 |
+
out = self.vae.encode(pixels[:, :, 0] if f == 1 else pixels).float()
|
| 227 |
+
if out.shape[2] != latent.shape[2]:
|
| 228 |
+
raise RuntimeError(f"VAE round trip changed latent frames {latent.shape[2]} -> {out.shape[2]}")
|
| 229 |
+
return out
|
| 230 |
+
|
| 231 |
+
def __call__(self, latent: torch.Tensor) -> torch.Tensor:
|
| 232 |
+
return self._resample(latent, scale_factor=2, mode="bicubic", align_corners=False)
|
| 233 |
+
|
| 234 |
+
def down(self, latent: torch.Tensor) -> torch.Tensor:
|
| 235 |
+
"""``D``: decode, area x0.5, encode (the LR latent of a cached HR item)."""
|
| 236 |
+
return self._resample(latent, scale_factor=0.5, mode="area")
|
| 237 |
+
|
| 238 |
+
|
| 239 |
+
def mask_to_latent(mask: torch.Tensor, scale: int, feather: float = 0.0) -> torch.Tensor:
|
| 240 |
+
"""LR-token mask ``(T, h, w)`` -> ``(1, 1, T, h*scale, w*scale)`` float, optionally feathered inward.
|
| 241 |
+
|
| 242 |
+
The feather is a ramp inside the region only (outside it the HR track was never denoised): weight
|
| 243 |
+
``1/R`` on the edge cell, rising by ``1/R`` per cell of depth, 1 from depth ``R - 1`` on. Every region
|
| 244 |
+
cell keeps weight > 0, and a region at least ``2R`` cells across reaches full weight in its middle.
|
| 245 |
+
(The old erode-then-blur left 40% of a 30% box below full weight and zeroed regions 2 tokens wide.)"""
|
| 246 |
+
up = mask.float().repeat_interleave(scale, 1).repeat_interleave(scale, 2)
|
| 247 |
+
radius = int(round(feather))
|
| 248 |
+
if radius > 1:
|
| 249 |
+
level = up[:, None]
|
| 250 |
+
total = level.clone()
|
| 251 |
+
for _ in range(radius - 1):
|
| 252 |
+
level = -F.max_pool2d(-level, 3, stride=1, padding=1) # erode one cell (edge of frame counts as inside)
|
| 253 |
+
total = total + level
|
| 254 |
+
up = (total / radius)[:, 0]
|
| 255 |
+
return up[None, None]
|
| 256 |
+
|
| 257 |
+
|
| 258 |
+
# ------------------------------------------------------------------------- sampler
|
| 259 |
+
|
| 260 |
+
|
| 261 |
+
def box_mask(box, frames: int, height: int, width: int) -> torch.Tensor:
|
| 262 |
+
x0, y0, x1, y1 = box
|
| 263 |
+
mask = torch.zeros(frames, height, width, dtype=torch.bool)
|
| 264 |
+
mask[:, round(y0 * height):round(y1 * height), round(x0 * width):round(x1 * width)] = True
|
| 265 |
+
return mask
|
| 266 |
+
|
| 267 |
+
|
| 268 |
+
@torch.no_grad()
|
| 269 |
+
def native_sample(dit, text, *, height, width, steps, seed, stop=None, capture_at=None, block=MID_BLOCK,
|
| 270 |
+
res_embed=None, latent_t=1):
|
| 271 |
+
"""Base-model Euler sampling; returns ``(x, x0_last, states)`` after ``stop`` steps (default all)."""
|
| 272 |
+
from h3_musubi import sigma_schedule
|
| 273 |
+
|
| 274 |
+
gen = torch.Generator("cuda").manual_seed(seed)
|
| 275 |
+
x = torch.randn(1, 24, latent_t, height // 16, width // 16, device="cuda", generator=gen)
|
| 276 |
+
audio_gen = torch.Generator("cuda").manual_seed(seed + 1)
|
| 277 |
+
sigmas = sigma_schedule(steps)
|
| 278 |
+
x0 = states = None
|
| 279 |
+
for index in range(steps if stop is None else stop):
|
| 280 |
+
sigma, sigma_next = float(sigmas[index]), float(sigmas[index + 1])
|
| 281 |
+
capture = {"block": block} if capture_at == index else None
|
| 282 |
+
velocity, _ = two_track_forward(dit, x, None, None, 1.0 - sigma, text, audio_generator=audio_gen,
|
| 283 |
+
capture=capture, res_embed=res_embed)
|
| 284 |
+
if capture is not None:
|
| 285 |
+
states = capture["states"]
|
| 286 |
+
x0 = x + sigma * velocity.float()
|
| 287 |
+
x = x0 + (sigma_next / sigma) * (x - x0)
|
| 288 |
+
return x, x0, states
|
| 289 |
+
|
| 290 |
+
|
| 291 |
+
@torch.no_grad()
|
| 292 |
+
def two_track_sample(dit, text, upscale, *, height, width, steps, switch, seed, maps=None, head=None, box=None,
|
| 293 |
+
block=MID_BLOCK, alpha=0.5, floor=None, feather=0.0, on_mask=None, hr_steps=None,
|
| 294 |
+
res_embed=None, decoder=None, latent_t=1, switch_upscale="vae", final="latent"):
|
| 295 |
+
"""The four-stage sampler of the module docstring. Returns ``(latent, info)`` at ``width x height``.
|
| 296 |
+
|
| 297 |
+
``decoder`` is ``(U, G)`` from ``train_tt.py decoder``: ``U`` replaces the VAE upscale (at the
|
| 298 |
+
switch and at the end) and ``G`` merges the tracks (their eqs. 5-6) instead of the feathered paste."""
|
| 299 |
+
from h3_musubi import sigma_schedule
|
| 300 |
+
|
| 301 |
+
lat_h, lat_w = height // 32, width // 32 # LR latent; also the HR token grid
|
| 302 |
+
tok_h, tok_w = lat_h // 2, lat_w // 2 # LR token grid
|
| 303 |
+
sigmas = sigma_schedule(steps)
|
| 304 |
+
lr, x0_lr, states = native_sample(dit, text, height=height // 2, width=width // 2, steps=steps, seed=seed,
|
| 305 |
+
stop=switch, capture_at=switch - 1 if head is not None else None, block=block,
|
| 306 |
+
res_embed=res_embed, latent_t=latent_t)
|
| 307 |
+
up = upscale if decoder is None else decoder[0]
|
| 308 |
+
if head is not None:
|
| 309 |
+
scores = head(states).float().cpu().reshape(latent_t, tok_h, tok_w)
|
| 310 |
+
mask = region_mask(scores, alpha=alpha, floor=floor)
|
| 311 |
+
else:
|
| 312 |
+
mask = box_mask(box or (0.25, 0.25, 0.75, 0.75), latent_t, tok_h, tok_w)
|
| 313 |
+
if on_mask is not None:
|
| 314 |
+
on_mask(mask)
|
| 315 |
+
tokens = [tok_h * tok_w * latent_t] * switch
|
| 316 |
+
if not mask.any() or switch >= steps:
|
| 317 |
+
# No region: finish the LR schedule with the base model, then upscale the clean latent.
|
| 318 |
+
lr_gen = torch.Generator("cuda").manual_seed(seed + 3)
|
| 319 |
+
for index in range(switch, steps):
|
| 320 |
+
sigma, sigma_next = float(sigmas[index]), float(sigmas[index + 1])
|
| 321 |
+
velocity, _ = two_track_forward(dit, lr, None, None, 1.0 - sigma, text, audio_generator=lr_gen,
|
| 322 |
+
res_embed=res_embed)
|
| 323 |
+
x0 = lr + sigma * velocity.float()
|
| 324 |
+
lr = x0 + (sigma_next / sigma) * (lr - x0)
|
| 325 |
+
tokens.append(tok_h * tok_w * latent_t)
|
| 326 |
+
return up(lr), {"mask": mask, "tokens": tokens}
|
| 327 |
+
plan = TwoTrackPlan(mask).to("cuda")
|
| 328 |
+
gen = torch.Generator("cuda").manual_seed(seed + 2)
|
| 329 |
+
audio_gen = torch.Generator("cuda").manual_seed(seed + 3)
|
| 330 |
+
sigma = float(sigmas[switch])
|
| 331 |
+
# At the switch the HR latent is mostly noise (sigma 0.75 at 16 of 20 on shift 12), so a latent x2 interpolation
|
| 332 |
+
# can stand in for the VAE round trip (5 s for 7 latent frames on a 5090): switch_upscale="latent".
|
| 333 |
+
# switch_upscale="crop": the HR track only reads U(x0_lr) inside M, so round-trip just M's bounding box
|
| 334 |
+
# (all frames, plus a margin) and fill the rest with a latent x2.
|
| 335 |
+
if switch_upscale == "vae":
|
| 336 |
+
start = up(x0_lr)
|
| 337 |
+
else:
|
| 338 |
+
start = F.interpolate(x0_lr.float(), scale_factor=(1, 2, 2), mode="trilinear", align_corners=False)
|
| 339 |
+
if switch_upscale == "crop":
|
| 340 |
+
box_lr = crop_box(mask, lat_h, lat_w, margin=2)
|
| 341 |
+
if box_lr is not None:
|
| 342 |
+
y0, y1, x0, x1 = box_lr
|
| 343 |
+
start[..., 2 * y0:2 * y1, 2 * x0:2 * x1] = up(x0_lr[..., y0:y1, x0:x1].contiguous())
|
| 344 |
+
hr = (1.0 - sigma) * start + sigma * torch.randn(1, 24, latent_t, 2 * lat_h, 2 * lat_w, device="cuda",
|
| 345 |
+
generator=gen)
|
| 346 |
+
# After the switch both tracks share the schedule: the model's own, or ``hr_steps`` steps
|
| 347 |
+
# linear in sigma (H3's shift 12 leaves few low-sigma steps, where the HR detail is made).
|
| 348 |
+
tail = sigmas[switch:] if not hr_steps else torch.linspace(sigma, 0.0, hr_steps + 1, dtype=torch.float64)
|
| 349 |
+
for index in range(len(tail) - 1):
|
| 350 |
+
sigma, sigma_next = float(tail[index]), float(tail[index + 1])
|
| 351 |
+
v_lr, v_hr = two_track_forward(dit, lr, hr, plan, 1.0 - sigma, text, maps=maps, audio_generator=audio_gen,
|
| 352 |
+
res_embed=res_embed)
|
| 353 |
+
ratio = sigma_next / sigma
|
| 354 |
+
lr = lr + (1.0 - ratio) * sigma * v_lr.float()
|
| 355 |
+
hr = hr + (1.0 - ratio) * sigma * v_hr.float()
|
| 356 |
+
tokens.append(plan.lr_count + plan.hr_count)
|
| 357 |
+
if decoder is not None:
|
| 358 |
+
upscaler, mixer = decoder
|
| 359 |
+
inside = mask_to_latent(mask, 2).to("cuda") # LR latent grid
|
| 360 |
+
# Outside M the HR latent is still switch-sigma noise (v_hr is zero there). D only feeds M, but the
|
| 361 |
+
# VAE's receptive field reaches past the edge, so fill the outside with a latent x2 of the LR track.
|
| 362 |
+
inside_hr = mask_to_latent(mask, 4).to("cuda")
|
| 363 |
+
context = F.interpolate(lr.float(), scale_factor=(1, 2, 2), mode="trilinear", align_corners=False)
|
| 364 |
+
clean_hr = inside_hr * hr + (1.0 - inside_hr) * context
|
| 365 |
+
filled = (1.0 - inside) * lr + inside * upscale.down(clean_hr) # their eq. 5: fill M with D(z_h)
|
| 366 |
+
latent = mixer(hr, upscaler(filled), mask_to_latent(mask, 4).to("cuda"))
|
| 367 |
+
return latent, {"mask": mask, "tokens": tokens}
|
| 368 |
+
weight = mask_to_latent(mask, 4, feather).to("cuda")
|
| 369 |
+
if final == "pixel":
|
| 370 |
+
# No round trip at the end: the caller decodes HR (outside M filled with a latent x2 of LR, so the
|
| 371 |
+
# VAE sees a plausible neighbourhood at the edge) and LR at half size, and pastes in pixel space.
|
| 372 |
+
inside = mask_to_latent(mask, 4).to("cuda")
|
| 373 |
+
context = F.interpolate(lr.float(), scale_factor=(1, 2, 2), mode="trilinear", align_corners=False)
|
| 374 |
+
return inside * hr + (1.0 - inside) * context, {"mask": mask, "tokens": tokens, "lr": lr, "weight": weight}
|
| 375 |
+
latent = weight * hr + (1.0 - weight) * upscale(lr)
|
| 376 |
+
return latent, {"mask": mask, "tokens": tokens}
|
| 377 |
+
|
| 378 |
+
|
| 379 |
+
def crop_box(mask: torch.Tensor, lat_h: int, lat_w: int, margin: int = 2):
|
| 380 |
+
"""Union over frames of M's bounding box on the LR latent grid (one LR token = 2 latent px), grown by
|
| 381 |
+
``margin`` latent px and rounded out to even sides, as ``(y0, y1, x0, x1)``; None for an empty mask."""
|
| 382 |
+
rows = torch.nonzero(mask.any(0).any(1)).flatten()
|
| 383 |
+
cols = torch.nonzero(mask.any(0).any(0)).flatten()
|
| 384 |
+
if not len(rows):
|
| 385 |
+
return None
|
| 386 |
+
y0, y1 = max(0, 2 * int(rows[0]) - margin), min(lat_h, 2 * int(rows[-1]) + 2 + margin)
|
| 387 |
+
x0, x1 = max(0, 2 * int(cols[0]) - margin), min(lat_w, 2 * int(cols[-1]) + 2 + margin)
|
| 388 |
+
y0, x0 = y0 - y0 % 2, x0 - x0 % 2
|
| 389 |
+
y1, x1 = min(lat_h, y1 + y1 % 2), min(lat_w, x1 + x1 % 2)
|
| 390 |
+
return y0, y1, x0, x1
|
| 391 |
+
|
| 392 |
+
|
| 393 |
+
def pixel_compose(vae, hr_latent: torch.Tensor, lr_latent: torch.Tensor, weight: torch.Tensor) -> torch.Tensor:
|
| 394 |
+
"""Decode HR and LR, upscale the LR pixels x2 (bicubic, per frame), paste with ``weight`` (HR latent grid,
|
| 395 |
+
``(1, 1, T, H, W)``) mapped to pixels: x16 in space, each latent frame to the pixel frames it decodes to."""
|
| 396 |
+
from h3_musubi import pixel_frames
|
| 397 |
+
|
| 398 |
+
# Long clips: each decode leaves the GPU as fp16 at once, and the paste runs ``chunk`` frames at a time,
|
| 399 |
+
# so the GPU never holds more than one full clip (16 s at 1344x768 is 2.4 GB in fp16, 4.8 GB in fp32).
|
| 400 |
+
dtype = next(vae.parameters()).dtype
|
| 401 |
+
hr_px = vae.decode(hr_latent.to("cuda", dtype)).clamp(-1, 1).to("cpu", torch.float16)
|
| 402 |
+
lr_px = vae.decode(lr_latent.to("cuda", dtype)).clamp(-1, 1).to("cpu", torch.float16)
|
| 403 |
+
if hr_latent.shape[2] == 1:
|
| 404 |
+
hr_px, lr_px = hr_px[:, :, :1], lr_px[:, :, :1]
|
| 405 |
+
frames = hr_px.shape[2]
|
| 406 |
+
frame_of = [k for k in range(hr_latent.shape[2]) for _ in range(pixel_frames(k + 1) - pixel_frames(k))]
|
| 407 |
+
frame_of = (frame_of + [frame_of[-1]] * frames)[:frames]
|
| 408 |
+
weight = weight[0, 0].float().to("cuda") # (T, H, W) on the HR latent grid
|
| 409 |
+
out = torch.empty_like(hr_px)
|
| 410 |
+
chunk = 16
|
| 411 |
+
for start in range(0, frames, chunk):
|
| 412 |
+
stop = min(frames, start + chunk)
|
| 413 |
+
lr = lr_px[0, :, start:stop].to("cuda", torch.float32).transpose(0, 1) # (n, 3, h, w)
|
| 414 |
+
lr_up = F.interpolate(lr, scale_factor=2, mode="bicubic", align_corners=False).clamp(-1, 1)
|
| 415 |
+
w_px = F.interpolate(weight[frame_of[start:stop]][:, None], scale_factor=16, mode="bilinear",
|
| 416 |
+
align_corners=False) # (n, 1, H, W)
|
| 417 |
+
hr = hr_px[0, :, start:stop].to("cuda", torch.float32).transpose(0, 1)
|
| 418 |
+
out[0, :, start:stop] = (w_px * hr + (1.0 - w_px) * lr_up).transpose(0, 1).to("cpu", torch.float16)
|
| 419 |
+
return out
|
| 420 |
+
|
| 421 |
+
|
| 422 |
+
# -------------------------------------------------------------------------- phases
|
| 423 |
+
|
| 424 |
+
|
| 425 |
+
def load_base(swap: int):
|
| 426 |
+
from h3_musubi import load_dit
|
| 427 |
+
|
| 428 |
+
return load_dit(CHECKPOINT, device="cuda", swap=swap)
|
| 429 |
+
|
| 430 |
+
|
| 431 |
+
def load_maps(path: Path) -> UpMaps:
|
| 432 |
+
"""``up_maps_tt.pt`` from ``fit-kv``, or the published ``up_maps_tt.safetensors`` (same keys, layers /
|
| 433 |
+
rank in the metadata)."""
|
| 434 |
+
path = Path(path)
|
| 435 |
+
if path.suffix == ".safetensors":
|
| 436 |
+
from safetensors import safe_open
|
| 437 |
+
from safetensors.torch import load_file
|
| 438 |
+
|
| 439 |
+
with safe_open(str(path), "pt") as handle:
|
| 440 |
+
meta = handle.metadata() or {}
|
| 441 |
+
saved = {"state": load_file(str(path)), "layers": int(meta["layers"]), "rank": int(meta["rank"])}
|
| 442 |
+
else:
|
| 443 |
+
saved = torch.load(path, map_location="cpu", weights_only=False)
|
| 444 |
+
maps = UpMaps(saved["layers"], HEAD_DIM, [EXTENT], rank=saved["rank"])
|
| 445 |
+
maps.load_state_dict(saved["state"])
|
| 446 |
+
return maps.to("cuda").eval()
|
| 447 |
+
|
| 448 |
+
|
| 449 |
+
def load_trained(dit, train_dir: Path, tag: str = ""):
|
| 450 |
+
"""LoRA (on), up-maps and resolution embedding from a ``train_tt.py lora`` run. Returns ``(network, maps, res_embed)``."""
|
| 451 |
+
from safetensors import safe_open
|
| 452 |
+
|
| 453 |
+
from h3_musubi import make_lora
|
| 454 |
+
|
| 455 |
+
with safe_open(str(train_dir / f"lora{tag}.safetensors"), "pt") as handle:
|
| 456 |
+
rank = int((handle.metadata() or {}).get("rank", 16))
|
| 457 |
+
network = make_lora(dit, rank)
|
| 458 |
+
info = network.load_weights(str(train_dir / f"lora{tag}.safetensors"))
|
| 459 |
+
if info.missing_keys:
|
| 460 |
+
raise RuntimeError(f"LoRA load missed {len(info.missing_keys)} keys")
|
| 461 |
+
network.to(device="cuda", dtype=torch.bfloat16).eval()
|
| 462 |
+
published = train_dir / f"tt_state{tag}.safetensors" # as upload_tt_hf.py publishes it
|
| 463 |
+
if published.is_file():
|
| 464 |
+
from safetensors.torch import load_file
|
| 465 |
+
|
| 466 |
+
with safe_open(str(published), "pt") as handle:
|
| 467 |
+
meta = handle.metadata() or {}
|
| 468 |
+
flat = load_file(str(published))
|
| 469 |
+
saved = {"layers": int(meta["layers"]), "rank": int(meta["rank"]), "hidden": int(meta["hidden"]),
|
| 470 |
+
"maps": {k[5:]: v for k, v in flat.items() if k.startswith("maps.")},
|
| 471 |
+
"res_embed": {k[10:]: v for k, v in flat.items() if k.startswith("res_embed.")}}
|
| 472 |
+
else:
|
| 473 |
+
saved = torch.load(train_dir / f"tt_state{tag}.pt", map_location="cpu", weights_only=False)
|
| 474 |
+
maps = UpMaps(saved["layers"], HEAD_DIM, [EXTENT], rank=saved["rank"])
|
| 475 |
+
maps.load_state_dict(saved["maps"])
|
| 476 |
+
res_embed = ResolutionEmbedding(saved["hidden"])
|
| 477 |
+
res_embed.load_state_dict(saved["res_embed"])
|
| 478 |
+
return network, maps.to("cuda").eval(), res_embed.to("cuda").eval()
|
| 479 |
+
|
| 480 |
+
|
| 481 |
+
def load_region(path: Path) -> dict:
|
| 482 |
+
"""``region_head_tt.pt`` (as ``train_tt.py region`` saves it) or ``.safetensors`` (as published)."""
|
| 483 |
+
path = Path(path)
|
| 484 |
+
if path.suffix == ".safetensors":
|
| 485 |
+
from safetensors import safe_open
|
| 486 |
+
from safetensors.torch import load_file
|
| 487 |
+
|
| 488 |
+
with safe_open(str(path), "pt") as handle:
|
| 489 |
+
meta = handle.metadata() or {}
|
| 490 |
+
return {"state": load_file(str(path)), "hidden": int(meta["hidden"]), "block": int(meta["block"])}
|
| 491 |
+
return torch.load(path, map_location="cpu", weights_only=False)
|
| 492 |
+
|
| 493 |
+
|
| 494 |
+
def load_decoder(path: Path):
|
| 495 |
+
from mixed_res import LatentUpscaler, TwoTrackMixer
|
| 496 |
+
|
| 497 |
+
saved = torch.load(path, map_location="cpu", weights_only=False)
|
| 498 |
+
upscaler, mixer = LatentUpscaler(width=saved["up_width"]), TwoTrackMixer(width=saved["mix_width"])
|
| 499 |
+
upscaler.load_state_dict(saved["upscaler"])
|
| 500 |
+
mixer.load_state_dict(saved["mixer"])
|
| 501 |
+
return upscaler.cuda().eval(), mixer.cuda().eval()
|
| 502 |
+
|
| 503 |
+
|
| 504 |
+
def load_upscaler() -> VaeUpscaler:
|
| 505 |
+
from h3_musubi import load_vae
|
| 506 |
+
from lot_paths import LOT_H3_VAE
|
| 507 |
+
|
| 508 |
+
return VaeUpscaler(load_vae(LOT_H3_VAE))
|
| 509 |
+
|
| 510 |
+
|
| 511 |
+
def held_prompts(args):
|
| 512 |
+
from render_h3 import load_prompts
|
| 513 |
+
from train_h3 import CACHE, list_items
|
| 514 |
+
|
| 515 |
+
if getattr(args, "te", None):
|
| 516 |
+
return load_prompts(args.prompts, list(args.te))
|
| 517 |
+
held = [te for _latent, te in list_items(args.cache or CACHE, args.holdout)[1]]
|
| 518 |
+
return load_prompts(args.prompts, held)
|
| 519 |
+
|
| 520 |
+
|
| 521 |
+
@torch.no_grad()
|
| 522 |
+
def render(args) -> None:
|
| 523 |
+
from PIL import Image, ImageDraw
|
| 524 |
+
from h3_musubi import sample
|
| 525 |
+
from render_h3 import to_image
|
| 526 |
+
|
| 527 |
+
from h3_musubi import set_lora
|
| 528 |
+
|
| 529 |
+
dit = load_base(args.swap)
|
| 530 |
+
upscale = load_upscaler()
|
| 531 |
+
maps = load_maps(args.maps) if args.maps else None
|
| 532 |
+
network = res_embed = None
|
| 533 |
+
if args.train:
|
| 534 |
+
network, maps, res_embed = load_trained(dit, args.train, args.tag)
|
| 535 |
+
decoder = load_decoder(args.decoder) if args.decoder else None
|
| 536 |
+
head = None
|
| 537 |
+
if args.region:
|
| 538 |
+
saved = load_region(args.region)
|
| 539 |
+
head = RegionHead(saved["hidden"]).cuda().eval()
|
| 540 |
+
head.load_state_dict(saved["state"])
|
| 541 |
+
args.block = saved["block"]
|
| 542 |
+
if args.train and saved.get("train") in (None, "None", ""):
|
| 543 |
+
print("LOT_H3 kind=warning msg=region head was trained on the base model's states, but --train turns the "
|
| 544 |
+
"LoRA on for the LR steps too; retrain it with train_tt.py region --train", flush=True)
|
| 545 |
+
from h3_musubi import latent_frames
|
| 546 |
+
width, height = args.size
|
| 547 |
+
latent_t = latent_frames(args.frames)
|
| 548 |
+
columns = [c for c in ("hr", "lr", "two_track") if c in args.variants]
|
| 549 |
+
meta = {"timings": {c: [] for c in columns}, "tokens": {c: [] for c in columns}, "hr_share": []}
|
| 550 |
+
args.out.mkdir(parents=True, exist_ok=True)
|
| 551 |
+
rows = []
|
| 552 |
+
for index, (caption, embeds) in enumerate(held_prompts(args)):
|
| 553 |
+
embeds = embeds.to("cuda", torch.bfloat16)
|
| 554 |
+
row = []
|
| 555 |
+
for name in columns:
|
| 556 |
+
mask = None
|
| 557 |
+
torch.cuda.synchronize()
|
| 558 |
+
start = time.perf_counter()
|
| 559 |
+
if network is not None:
|
| 560 |
+
set_lora(network, name == "two_track") # the baselines are the base model
|
| 561 |
+
if name == "hr":
|
| 562 |
+
latent = sample(dit, embeds, height=height, width=width, steps=args.steps, seed=args.seed + index,
|
| 563 |
+
latent_t=latent_t)
|
| 564 |
+
count = float((height // 32) * (width // 32) * latent_t)
|
| 565 |
+
elif name == "lr":
|
| 566 |
+
latent, _x0, _s = native_sample(dit, embeds, height=height // 2, width=width // 2, steps=args.steps,
|
| 567 |
+
seed=args.seed + index, latent_t=latent_t)
|
| 568 |
+
latent = upscale(latent)
|
| 569 |
+
count = float((height // 64) * (width // 64) * latent_t)
|
| 570 |
+
else:
|
| 571 |
+
latent, info = two_track_sample(
|
| 572 |
+
dit, embeds, upscale, height=height, width=width, steps=args.steps, switch=args.switch,
|
| 573 |
+
seed=args.seed + index, maps=maps, head=head, box=args.box, block=args.block,
|
| 574 |
+
alpha=args.alpha, floor=args.floor, feather=args.feather, hr_steps=args.hr_steps,
|
| 575 |
+
res_embed=res_embed, decoder=decoder, latent_t=latent_t, switch_upscale=args.switch_upscale,
|
| 576 |
+
final=args.final)
|
| 577 |
+
mask = info["mask"]
|
| 578 |
+
count = sum(info["tokens"]) / len(info["tokens"])
|
| 579 |
+
meta["hr_share"].append(round(float(mask.float().mean()), 4))
|
| 580 |
+
meta["tokens"][name].append(round(count, 1))
|
| 581 |
+
# Every variant is timed through its decode to pixels (sampling + VAE), the number a user waits for.
|
| 582 |
+
if name == "two_track" and args.final == "pixel" and "lr" in info:
|
| 583 |
+
pixels = pixel_compose(upscale.vae, latent, info["lr"], info["weight"])
|
| 584 |
+
else:
|
| 585 |
+
pixels = upscale.vae.decode(latent.to("cuda", upscale.dtype)).to("cpu", torch.float16)
|
| 586 |
+
torch.cuda.synchronize()
|
| 587 |
+
meta["timings"][name].append(round(time.perf_counter() - start, 2))
|
| 588 |
+
meta["note"] = "timings include the VAE decode for every variant"
|
| 589 |
+
if latent_t > 1:
|
| 590 |
+
save_clip(pixels, args.out / f"p{index}_{name}.mp4")
|
| 591 |
+
pixels = pixels[:, :, pixels.shape[2] // 2:pixels.shape[2] // 2 + 1] # the grid shows the middle frame
|
| 592 |
+
image = to_image(pixels)
|
| 593 |
+
if mask is not None and args.outline:
|
| 594 |
+
image = outline(image, mask[mask.shape[0] // 2:mask.shape[0] // 2 + 1])
|
| 595 |
+
image.save(args.out / f"p{index}_{name}.png")
|
| 596 |
+
row.append(image)
|
| 597 |
+
print(f"LOT_H3 kind=two_track_render prompt={index} variant={name} "
|
| 598 |
+
f"seconds={meta['timings'][name][-1]} mean_tokens={count:.0f}", flush=True)
|
| 599 |
+
rows.append(row)
|
| 600 |
+
scale = 0.5
|
| 601 |
+
cell = (int(width * scale), int(height * scale))
|
| 602 |
+
sheet = Image.new("RGB", (cell[0] * len(columns), 28 + cell[1] * len(rows)), "white")
|
| 603 |
+
draw = ImageDraw.Draw(sheet)
|
| 604 |
+
for col, label in enumerate(columns):
|
| 605 |
+
draw.text((col * cell[0] + 6, 8), label, fill="black")
|
| 606 |
+
for r, images in enumerate(rows):
|
| 607 |
+
for col, image in enumerate(images):
|
| 608 |
+
sheet.paste(image.resize(cell, Image.LANCZOS), (col * cell[0], 28 + r * cell[1]))
|
| 609 |
+
sheet.save(args.out / "grid.png")
|
| 610 |
+
meta.update({"columns": columns, "frames": args.frames, "latent_t": latent_t, "steps": args.steps, "switch": args.switch, "canvas": f"{width}x{height}",
|
| 611 |
+
"maps": str(args.maps), "train": str(args.train), "decoder": str(args.decoder), "region": str(args.region), "box": args.box, "feather": args.feather})
|
| 612 |
+
(args.out / "meta.json").write_text(json.dumps(meta, indent=1) + "\n")
|
| 613 |
+
print(f"LOT_H3 kind=two_track_render_done out={args.out} ok=1", flush=True)
|
| 614 |
+
|
| 615 |
+
|
| 616 |
+
@torch.no_grad()
|
| 617 |
+
def draw_layout(args) -> None:
|
| 618 |
+
"""What the region head picks, drawn: for held-out prompts from each ``--cache``, the base model samples
|
| 619 |
+
at half resolution; at ``--switch`` the head scores every token. Each row shows the final half-res
|
| 620 |
+
render (upscaled for display) with the score heatmap and the selected region, and the LoT layout that
|
| 621 |
+
region implies on the full-resolution token grid (1x1 inside, 2x2 on a one-token ring, 4x4 elsewhere)."""
|
| 622 |
+
import numpy as np
|
| 623 |
+
from PIL import Image, ImageDraw
|
| 624 |
+
|
| 625 |
+
from mixed_res import region_layout
|
| 626 |
+
from render_h3 import load_prompts
|
| 627 |
+
from train_h3 import list_items
|
| 628 |
+
|
| 629 |
+
dit = load_base(args.swap)
|
| 630 |
+
upscale = load_upscaler()
|
| 631 |
+
saved = load_region(args.region)
|
| 632 |
+
head = RegionHead(saved["hidden"]).cuda().eval()
|
| 633 |
+
head.load_state_dict(saved["state"])
|
| 634 |
+
block = saved["block"]
|
| 635 |
+
width, height = args.size
|
| 636 |
+
from safetensors import safe_open
|
| 637 |
+
|
| 638 |
+
from build_pairs_tt import caption_for
|
| 639 |
+
|
| 640 |
+
prompts = []
|
| 641 |
+
for cache in args.cache_list or [None]:
|
| 642 |
+
if cache is None:
|
| 643 |
+
prompts += load_prompts(args.prompts, None)
|
| 644 |
+
continue
|
| 645 |
+
for te in [te for _l, te in list_items(cache, args.holdout)[1]][:args.prompts]:
|
| 646 |
+
with safe_open(str(te), "pt") as handle:
|
| 647 |
+
hidden, keep = handle.get_tensor("hidden_states"), handle.get_tensor("attention_mask")
|
| 648 |
+
prompts.append((caption_for(str(te)) or te.name, hidden[keep].unsqueeze(0)))
|
| 649 |
+
colors = {1: (74, 222, 128), 4: (139, 180, 240), 16: (192, 132, 252)}
|
| 650 |
+
rows, meta = [], []
|
| 651 |
+
for index, (caption, embeds) in enumerate(prompts):
|
| 652 |
+
latent, _x0, states = native_sample(dit, embeds.to("cuda", torch.bfloat16), height=height // 2,
|
| 653 |
+
width=width // 2, steps=args.steps, seed=args.seed + index,
|
| 654 |
+
capture_at=args.switch - 1, block=block)
|
| 655 |
+
tok_h, tok_w = height // 64, width // 64
|
| 656 |
+
scores = head(states).float().cpu().reshape(1, tok_h, tok_w)
|
| 657 |
+
mask = region_mask(scores, alpha=args.alpha, floor=args.floor)
|
| 658 |
+
full = mask.repeat_interleave(2, 1).repeat_interleave(2, 2)
|
| 659 |
+
layout = region_layout(full, background=4, ring=1)
|
| 660 |
+
image = to_image_from(upscale.vae.decode(latent.to("cuda", upscale.dtype))).resize((width, height), Image.LANCZOS)
|
| 661 |
+
# Left: heatmap of the head's probability with the selected region outlined.
|
| 662 |
+
prob = torch.sigmoid(scores[0]).numpy()
|
| 663 |
+
heat = Image.fromarray((np.clip(prob, 0, 1) * 255).astype(np.uint8)).resize((width, height), Image.NEAREST)
|
| 664 |
+
red = Image.merge("RGB", (heat, Image.new("L", heat.size, 40), Image.new("L", heat.size, 40)))
|
| 665 |
+
left = outline(Image.blend(image, red, 0.45), mask)
|
| 666 |
+
# Right: the LoT layout, every token drawn at full-resolution token size (32 px).
|
| 667 |
+
right = image.copy()
|
| 668 |
+
draw = ImageDraw.Draw(right, "RGBA")
|
| 669 |
+
for rect in layout.rects:
|
| 670 |
+
x0, y0 = rect.v * 32, rect.u * 32
|
| 671 |
+
area = rect.eh * rect.ew
|
| 672 |
+
r, g, b = colors.get(area, (167, 139, 250))
|
| 673 |
+
draw.rectangle([x0, y0, x0 + rect.ew * 32 - 1, y0 + rect.eh * 32 - 1], outline=(r, g, b, 230),
|
| 674 |
+
fill=(r, g, b, 40 if area > 1 else 70))
|
| 675 |
+
row = Image.new("RGB", (2 * width, height + 30), "white")
|
| 676 |
+
row.paste(left, (0, 30))
|
| 677 |
+
row.paste(right, (width, 30))
|
| 678 |
+
label = f"{caption[:110]} | region {float(mask.float().mean()):.0%} of frame, layout {layout.count} of {tok_h * tok_w * 4} tokens"
|
| 679 |
+
ImageDraw.Draw(row).text((8, 9), label, fill="black")
|
| 680 |
+
rows.append(row)
|
| 681 |
+
meta.append({"caption": caption, "region_share": round(float(mask.float().mean()), 4),
|
| 682 |
+
"layout_tokens": layout.count, "full_tokens": tok_h * tok_w * 4})
|
| 683 |
+
print(f"LOT_H3 kind=layout prompt={index} region={float(mask.float().mean()):.2f} tokens={layout.count}",
|
| 684 |
+
flush=True)
|
| 685 |
+
args.out.mkdir(parents=True, exist_ok=True)
|
| 686 |
+
scale = args.sheet_scale
|
| 687 |
+
cell = (int(2 * width * scale), int((height + 30) * scale))
|
| 688 |
+
for start in range(0, len(rows), args.per_sheet):
|
| 689 |
+
chunk = rows[start:start + args.per_sheet]
|
| 690 |
+
sheet = Image.new("RGB", (cell[0], cell[1] * len(chunk)), "white")
|
| 691 |
+
for number, row in enumerate(chunk):
|
| 692 |
+
sheet.paste(row.resize(cell, Image.LANCZOS), (0, number * cell[1]))
|
| 693 |
+
sheet.save(args.out / f"layout_{start // args.per_sheet}.jpg", quality=90)
|
| 694 |
+
(args.out / "layout.json").write_text(json.dumps({"switch": args.switch, "alpha": args.alpha,
|
| 695 |
+
"region": str(args.region), "items": meta}, indent=1) + "\n")
|
| 696 |
+
print(f"LOT_H3 kind=layout_done out={args.out} prompts={len(rows)} ok=1", flush=True)
|
| 697 |
+
|
| 698 |
+
|
| 699 |
+
def save_clip(pixels: torch.Tensor, path: Path, fps: int = 24) -> None:
|
| 700 |
+
"""(1, 3, F, H, W) in [-1, 1] -> mp4."""
|
| 701 |
+
import imageio
|
| 702 |
+
|
| 703 |
+
frames = ((pixels[0].float().clamp(-1, 1) + 1) * 127.5).round().byte().permute(1, 2, 3, 0).cpu().numpy()
|
| 704 |
+
imageio.mimwrite(str(path), list(frames), fps=fps, quality=8, macro_block_size=1)
|
| 705 |
+
|
| 706 |
+
|
| 707 |
+
def to_image_from(pixels):
|
| 708 |
+
from render_h3 import to_image
|
| 709 |
+
|
| 710 |
+
return to_image(pixels)
|
| 711 |
+
|
| 712 |
+
|
| 713 |
+
def outline(image, mask: torch.Tensor):
|
| 714 |
+
"""Draw the region's boundary (one LR token is 64 px)."""
|
| 715 |
+
from PIL import ImageDraw
|
| 716 |
+
|
| 717 |
+
image = image.copy()
|
| 718 |
+
draw = ImageDraw.Draw(image)
|
| 719 |
+
grid = mask[0]
|
| 720 |
+
cell = image.height // grid.shape[0]
|
| 721 |
+
for u in range(grid.shape[0]):
|
| 722 |
+
for v in range(grid.shape[1]):
|
| 723 |
+
if not grid[u, v]:
|
| 724 |
+
continue
|
| 725 |
+
x, y = v * cell, u * cell
|
| 726 |
+
if u == 0 or not grid[u - 1, v]:
|
| 727 |
+
draw.line([(x, y), (x + cell, y)], fill="white", width=2)
|
| 728 |
+
if u == grid.shape[0] - 1 or not grid[u + 1, v]:
|
| 729 |
+
draw.line([(x, y + cell), (x + cell, y + cell)], fill="white", width=2)
|
| 730 |
+
if v == 0 or not grid[u, v - 1]:
|
| 731 |
+
draw.line([(x, y), (x, y + cell)], fill="white", width=2)
|
| 732 |
+
if v == grid.shape[1] - 1 or not grid[u, v + 1]:
|
| 733 |
+
draw.line([(x + cell, y), (x + cell, y + cell)], fill="white", width=2)
|
| 734 |
+
return image
|
| 735 |
+
|
| 736 |
+
|
| 737 |
+
def train_items(args):
|
| 738 |
+
from train_h3 import CACHE, list_items
|
| 739 |
+
|
| 740 |
+
train, _held = list_items(args.cache or CACHE, args.holdout)
|
| 741 |
+
random.Random(args.seed).shuffle(train)
|
| 742 |
+
return train[:args.items]
|
| 743 |
+
|
| 744 |
+
|
| 745 |
+
def crop_even(x0: torch.Tensor) -> torch.Tensor:
|
| 746 |
+
"""Crop a cached latent so its LR version has an even latent size (HR latent sides multiple of 4)."""
|
| 747 |
+
height, width = x0.shape[-2] // 4 * 4, x0.shape[-1] // 4 * 4
|
| 748 |
+
return x0[..., :height, :width]
|
| 749 |
+
|
| 750 |
+
|
| 751 |
+
@torch.no_grad()
|
| 752 |
+
def fit_kv(args) -> None:
|
| 753 |
+
"""Ridge fit of ``f_up`` (their Sec. 3.1): the base model's keys/values at LR and at HR on the
|
| 754 |
+
same images and noise level, each LR token paired with its four HR tokens."""
|
| 755 |
+
from train_h3 import load_item
|
| 756 |
+
|
| 757 |
+
dit = load_base(args.swap)
|
| 758 |
+
layers = len(dit.blocks)
|
| 759 |
+
maps = UpMaps(layers, HEAD_DIM, [EXTENT], rank=args.rank).cuda()
|
| 760 |
+
moments: dict = {}
|
| 761 |
+
if args.pairs:
|
| 762 |
+
from build_pairs_tt import list_pairs, load_pair
|
| 763 |
+
|
| 764 |
+
sources = [("pair", entry) for entry in list_pairs(args.pairs, 8)[0][:args.items]]
|
| 765 |
+
else:
|
| 766 |
+
upscale = load_upscaler()
|
| 767 |
+
sources = [("cache", item) for item in train_items(args)]
|
| 768 |
+
for index, (kind, item) in enumerate(sources):
|
| 769 |
+
if kind == "pair":
|
| 770 |
+
pair = load_pair(item)
|
| 771 |
+
hr0, lr0, text = pair["hr"], pair["lr"], pair["text"]
|
| 772 |
+
else:
|
| 773 |
+
x0, text = load_item(item, random.Random(index), "cuda")
|
| 774 |
+
hr0 = crop_even(x0)
|
| 775 |
+
lr0 = upscale.down(hr0)
|
| 776 |
+
plan = TwoTrackPlan(torch.zeros(1, lr0.shape[-2] // 2, lr0.shape[-1] // 2, dtype=torch.bool))
|
| 777 |
+
cells = torch.argsort(plan.cell_order).reshape(plan.lr_count, 4).cuda() # LR token -> its HR rows
|
| 778 |
+
for sigma in args.sigmas:
|
| 779 |
+
gen = torch.Generator("cuda").manual_seed(index)
|
| 780 |
+
kv_lr, kv_hr = {}, {}
|
| 781 |
+
x_lr = (1.0 - sigma) * lr0 + sigma * torch.randn(lr0.shape, device="cuda", generator=gen)
|
| 782 |
+
x_hr = (1.0 - sigma) * hr0 + sigma * torch.randn(hr0.shape, device="cuda", generator=gen)
|
| 783 |
+
two_track_forward(dit, x_lr, None, None, 1.0 - sigma, text, capture={"kv": kv_lr})
|
| 784 |
+
two_track_forward(dit, x_hr, None, None, 1.0 - sigma, text, capture={"kv": kv_hr})
|
| 785 |
+
for layer in kv_lr:
|
| 786 |
+
for which, slot in (("k", 0), ("v", 1)):
|
| 787 |
+
x = kv_lr[layer][slot][0].float().reshape(-1, HEAD_DIM) # (N*H, d)
|
| 788 |
+
y = kv_hr[layer][slot][0].float()[cells].transpose(1, 2).reshape(-1, 4, HEAD_DIM)
|
| 789 |
+
delta = (y - x.unsqueeze(1)).reshape(x.shape[0], -1)
|
| 790 |
+
entry = moments.setdefault((layer, which), [0.0, 0.0, 0, 0.0, 0.0])
|
| 791 |
+
entry[0] = entry[0] + x.T @ x
|
| 792 |
+
entry[1] = entry[1] + x.T @ delta
|
| 793 |
+
entry[2] += x.shape[0]
|
| 794 |
+
entry[3] += float(delta.pow(2).sum()) # copy-init residual
|
| 795 |
+
entry[4] += float(y.pow(2).sum())
|
| 796 |
+
print(f"LOT_H3 kind=tt_fit_kv item={index}", flush=True)
|
| 797 |
+
report = {"k": [], "v": []}
|
| 798 |
+
for (layer, which), (xtx, xty, count, copy_err, energy) in moments.items():
|
| 799 |
+
maps.fit_moments_(layer, EXTENT, which, xtx, xty, count, args.ridge)
|
| 800 |
+
name = f"l{layer}_2x2_{which}"
|
| 801 |
+
weight = maps.down[name].float() @ maps.up[name].float()
|
| 802 |
+
# |Y - XW|^2 = |Y|^2 - 2 tr(W^T X^T Y) + tr(W^T X^T X W), on the training moments.
|
| 803 |
+
fit_err = copy_err - 2.0 * float((weight * xty).sum()) + float((weight * (xtx @ weight)).sum())
|
| 804 |
+
report[which].append((layer, (copy_err / energy) ** 0.5, (max(fit_err, 0.0) / energy) ** 0.5))
|
| 805 |
+
for which in ("k", "v"):
|
| 806 |
+
rows = sorted(report[which])
|
| 807 |
+
copy = sum(r[1] for r in rows) / len(rows)
|
| 808 |
+
fit = sum(r[2] for r in rows) / len(rows)
|
| 809 |
+
print(f"LOT_H3 kind=tt_fit_kv_residual which={which} copy_rel={copy:.3f} fit_rel={fit:.3f}", flush=True)
|
| 810 |
+
args.out.mkdir(parents=True, exist_ok=True)
|
| 811 |
+
torch.save({"state": maps.state_dict(), "rank": args.rank, "layers": layers, "sigmas": args.sigmas,
|
| 812 |
+
"residual": report}, args.out / "up_maps_tt.pt")
|
| 813 |
+
print(f"LOT_H3 kind=tt_fit_kv_done out={args.out / 'up_maps_tt.pt'} ok=1", flush=True)
|
| 814 |
+
|
| 815 |
+
|
| 816 |
+
def train_region(args) -> None:
|
| 817 |
+
"""Linear head on the LR track's block states at the switch. Label (their self-supervised one, in
|
| 818 |
+
latent space): the ``--fraction`` of LR tokens where the upscaled LR clean estimate is furthest
|
| 819 |
+
from the true HR latent."""
|
| 820 |
+
from h3_musubi import sigma_schedule
|
| 821 |
+
from train_h3 import load_item
|
| 822 |
+
|
| 823 |
+
dit = load_base(args.swap)
|
| 824 |
+
upscale = load_upscaler()
|
| 825 |
+
sigma = float(sigma_schedule(args.steps)[args.switch - 1])
|
| 826 |
+
pairs = []
|
| 827 |
+
with torch.no_grad():
|
| 828 |
+
for index, item in enumerate(train_items(args)):
|
| 829 |
+
x0, text = load_item(item, random.Random(index), "cuda")
|
| 830 |
+
hr0 = crop_even(x0)
|
| 831 |
+
lr0 = upscale.down(hr0)
|
| 832 |
+
gen = torch.Generator("cuda").manual_seed(index)
|
| 833 |
+
x_lr = (1.0 - sigma) * lr0 + sigma * torch.randn(lr0.shape, device="cuda", generator=gen)
|
| 834 |
+
capture = {"block": args.block}
|
| 835 |
+
velocity, _ = two_track_forward(dit, x_lr, None, None, 1.0 - sigma, text, capture=capture)
|
| 836 |
+
error = (upscale(x_lr + sigma * velocity.float()) - hr0).abs().mean(dim=1) # (1, 4h, 4w) latent px
|
| 837 |
+
error = F.avg_pool2d(error, 4).flatten() # LR tokens
|
| 838 |
+
label = (error >= torch.quantile(error, 1.0 - args.fraction)).float()
|
| 839 |
+
pairs.append((capture["states"].cpu(), label.cpu()))
|
| 840 |
+
print(f"LOT_H3 kind=tt_region_pair item={index} tokens={label.numel()}", flush=True)
|
| 841 |
+
del dit
|
| 842 |
+
torch.cuda.empty_cache()
|
| 843 |
+
head = RegionHead(pairs[0][0].shape[-1]).cuda()
|
| 844 |
+
optimizer = torch.optim.Adam(head.parameters(), lr=args.lr)
|
| 845 |
+
held = pairs[: max(1, len(pairs) // 10)]
|
| 846 |
+
train = pairs[len(held):] or pairs
|
| 847 |
+
for epoch in range(args.epochs):
|
| 848 |
+
random.Random(epoch).shuffle(train)
|
| 849 |
+
total = 0.0
|
| 850 |
+
for states, target in train:
|
| 851 |
+
loss = F.binary_cross_entropy_with_logits(head(states.cuda()), target.cuda())
|
| 852 |
+
optimizer.zero_grad(set_to_none=True)
|
| 853 |
+
loss.backward()
|
| 854 |
+
optimizer.step()
|
| 855 |
+
total += float(loss.detach())
|
| 856 |
+
with torch.no_grad():
|
| 857 |
+
held_bce = sum(float(F.binary_cross_entropy_with_logits(head(s.cuda()), t.cuda())) for s, t in held)
|
| 858 |
+
print(f"LOT_H3 kind=tt_region_epoch epoch={epoch} train_bce={total / len(train):.4f} "
|
| 859 |
+
f"held_bce={held_bce / len(held):.4f}", flush=True)
|
| 860 |
+
args.out.mkdir(parents=True, exist_ok=True)
|
| 861 |
+
torch.save({"state": head.state_dict(), "hidden": pairs[0][0].shape[-1], "block": args.block,
|
| 862 |
+
"switch": args.switch, "steps": args.steps, "fraction": args.fraction},
|
| 863 |
+
args.out / "region_head_tt.pt")
|
| 864 |
+
print(f"LOT_H3 kind=tt_region_done out={args.out / 'region_head_tt.pt'} ok=1", flush=True)
|
| 865 |
+
|
| 866 |
+
|
| 867 |
+
def main() -> None:
|
| 868 |
+
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
| 869 |
+
parser.add_argument("phase", choices=("render", "fit-kv", "region", "layout"))
|
| 870 |
+
parser.add_argument("--cache-list", type=Path, nargs="+", default=None,
|
| 871 |
+
help="layout: caches whose held-out captions are the prompts (--prompts from each)")
|
| 872 |
+
parser.add_argument("--per-sheet", type=int, default=6, help="layout: rows per output sheet")
|
| 873 |
+
parser.add_argument("--sheet-scale", type=float, default=0.5, help="layout: sheet size relative to 2x canvas")
|
| 874 |
+
parser.add_argument("--out", type=Path, required=True)
|
| 875 |
+
parser.add_argument("--cache", type=Path, default=None, help="still cache with text (prompts and fit data)")
|
| 876 |
+
parser.add_argument("--holdout", type=int, default=24)
|
| 877 |
+
parser.add_argument("--te", type=Path, nargs="+", default=None,
|
| 878 |
+
help="render: caption embeddings (*_minimaxh3_te.safetensors) to use as prompts instead of a "
|
| 879 |
+
"cache's held-out items, e.g. from the HF dataset johndpope/MiniMax-H3-pexels-people-text")
|
| 880 |
+
parser.add_argument("--items", type=int, default=24)
|
| 881 |
+
parser.add_argument("--prompts", type=int, default=3)
|
| 882 |
+
parser.add_argument("--seed", type=int, default=0)
|
| 883 |
+
parser.add_argument("--swap", default="auto", help="DiT blocks streamed from CPU; auto (by free VRAM): 0 at 64 GB+, 8 at 28+, 16 at 21+, else 24")
|
| 884 |
+
parser.add_argument("--steps", type=int, default=20)
|
| 885 |
+
parser.add_argument("--switch", type=int, default=18, help="LR-only steps before the HR region starts")
|
| 886 |
+
parser.add_argument("--hr-steps", type=int, default=6,
|
| 887 |
+
help="steps after the switch, linear in sigma (0: the rest of the schedule)")
|
| 888 |
+
parser.add_argument("--size", type=int, nargs=2, default=(1344, 768), metavar=("W", "H"))
|
| 889 |
+
parser.add_argument("--switch-upscale", choices=("vae", "crop", "latent"), default="vae",
|
| 890 |
+
help="render: HR start at the switch: VAE round trip of the frame, of M's bounding box only "
|
| 891 |
+
"(crop), or latent x2 interpolation (no VAE; striped regions)")
|
| 892 |
+
parser.add_argument("--preset", choices=("quality",), default=None,
|
| 893 |
+
help="render: quality = switch 16, 6 HR steps, crop upscale, pixel paste, and the fitted maps "
|
| 894 |
+
"($LOT_TT_MAPS) and region head ($LOT_TT_REGION) unless --maps / --region / --box are given")
|
| 895 |
+
parser.add_argument("--final", choices=("latent", "pixel"), default="latent",
|
| 896 |
+
help="render: merge the tracks in latent space (a VAE round trip of LR) or in pixel space "
|
| 897 |
+
"(decode LR at half size, paste after decoding)")
|
| 898 |
+
parser.add_argument("--frames", type=int, default=1,
|
| 899 |
+
help="render: pixel frames (1 = still; 22 = 7 latent frames). Clips also save p<i>_<variant>.mp4")
|
| 900 |
+
parser.add_argument("--variants", default="hr,lr,two_track")
|
| 901 |
+
parser.add_argument("--maps", type=Path, default=None, help="up_maps_tt.pt from fit-kv (default: copy)")
|
| 902 |
+
parser.add_argument("--train", type=Path, default=None,
|
| 903 |
+
help="train_tt.py lora output dir: LoRA + fine-tuned maps + resolution embedding (overrides --maps)")
|
| 904 |
+
parser.add_argument("--tag", default="", help="checkpoint tag in --train, e.g. _1000")
|
| 905 |
+
parser.add_argument("--decoder", type=Path, default=None, help="decoder.pt from train_tt.py decoder (U and G)")
|
| 906 |
+
parser.add_argument("--pairs", type=Path, action="append", default=None,
|
| 907 |
+
help="fit-kv: build_pairs_tt.py dirs (HR + LR latents precomputed; repeatable)")
|
| 908 |
+
parser.add_argument("--region", type=Path, default=None, help="region_head_tt.pt (default: --box)")
|
| 909 |
+
parser.add_argument("--box", type=float, nargs=4, default=None, metavar=("X0", "Y0", "X1", "Y1"))
|
| 910 |
+
parser.add_argument("--alpha", type=float, default=0.5)
|
| 911 |
+
parser.add_argument("--floor", type=float, default=None)
|
| 912 |
+
parser.add_argument("--feather", type=float, default=4.0,
|
| 913 |
+
help="paste-edge ramp inside the region, HR latent pixels (weight 1/R at the edge, 1 from R-1 deep)")
|
| 914 |
+
parser.add_argument("--outline", action="store_true", help="draw the region boundary on the two_track image")
|
| 915 |
+
parser.add_argument("--block", type=int, default=MID_BLOCK)
|
| 916 |
+
parser.add_argument("--fraction", type=float, default=0.25)
|
| 917 |
+
parser.add_argument("--epochs", type=int, default=20)
|
| 918 |
+
parser.add_argument("--lr", type=float, default=1e-3)
|
| 919 |
+
parser.add_argument("--rank", type=int, default=HEAD_DIM, help="rank of the f_up delta (head dim = full)")
|
| 920 |
+
parser.add_argument("--ridge", type=float, default=1e-2)
|
| 921 |
+
parser.add_argument("--sigmas", type=float, nargs="+", default=[0.2, 0.4, 0.6, 0.8, 0.95],
|
| 922 |
+
help="fit-kv: noise levels; cover the sigmas the HR track runs at after the switch")
|
| 923 |
+
args = parser.parse_args()
|
| 924 |
+
if args.preset == "quality":
|
| 925 |
+
args.switch, args.hr_steps, args.switch_upscale, args.final = 16, 6, "crop", "pixel"
|
| 926 |
+
if args.maps is None and args.train is None:
|
| 927 |
+
path = os.environ.get("LOT_TT_MAPS")
|
| 928 |
+
if not path or not Path(path).is_file():
|
| 929 |
+
raise SystemExit("--preset quality needs fitted maps: --maps up_maps_tt.pt (fit-kv) or $LOT_TT_MAPS; "
|
| 930 |
+
"without them the HR region has a diamond-grid grain")
|
| 931 |
+
args.maps = Path(path)
|
| 932 |
+
if args.region is None and args.box is None and os.environ.get("LOT_TT_REGION"):
|
| 933 |
+
args.region = Path(os.environ["LOT_TT_REGION"])
|
| 934 |
+
from gpu_guard import auto_swap
|
| 935 |
+
|
| 936 |
+
from h3_musubi import latent_frames as _latent_frames
|
| 937 |
+
args.swap = auto_swap(args.swap, inference=args.phase in ("render", "layout"),
|
| 938 |
+
latent_t=_latent_frames(getattr(args, "frames", 1)))
|
| 939 |
+
if args.size[0] % 64 or args.size[1] % 64:
|
| 940 |
+
raise SystemExit("--size sides must be multiples of 64 (the LR latent needs whole 2x2 patches)")
|
| 941 |
+
refuse_if_busy("two_track_h3")
|
| 942 |
+
{"render": render, "fit-kv": fit_kv, "region": train_region, "layout": draw_layout}[args.phase](args)
|
| 943 |
+
|
| 944 |
+
|
| 945 |
+
if __name__ == "__main__":
|
| 946 |
+
main()
|
requirements.txt
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Vendored lot/ + third_party/musubi_tuner subset pulls these on the load path:
|
| 2 |
+
# - av (minimax_h3/media.py + dataset/*, module-level)
|
| 3 |
+
# - opencv (dataset/media_utils.py via cv2_compat — headless, no GUI deps)
|
| 4 |
+
# - accelerate (init_empty_weights in model.py / video_vae.py)
|
| 5 |
+
# - imageio(+ffmpeg) for two_track_h3.save_clip's mp4 writer
|
| 6 |
+
# gradio, spaces, huggingface_hub and torch are preinstalled on ZeroGPU — not pinned here.
|
| 7 |
+
accelerate
|
| 8 |
+
safetensors>=0.8.0
|
| 9 |
+
numpy
|
| 10 |
+
pillow
|
| 11 |
+
tqdm
|
| 12 |
+
av
|
| 13 |
+
opencv-python-headless
|
| 14 |
+
imageio
|
| 15 |
+
imageio-ffmpeg
|
third_party/musubi_tuner/__init__.py
ADDED
|
File without changes
|
third_party/musubi_tuner/convert_lora.py
ADDED
|
@@ -0,0 +1,229 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import argparse
|
| 2 |
+
|
| 3 |
+
import torch
|
| 4 |
+
from safetensors.torch import load_file, save_file
|
| 5 |
+
from safetensors import safe_open
|
| 6 |
+
from musubi_tuner.utils import model_utils
|
| 7 |
+
|
| 8 |
+
import logging
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
logger = logging.getLogger(__name__)
|
| 12 |
+
logging.basicConfig(level=logging.INFO)
|
| 13 |
+
|
| 14 |
+
# keys of Qwen-Image state dict
|
| 15 |
+
QWEN_IMAGE_KEYS = [
|
| 16 |
+
"time_text_embed.timestep_embedder.linear_1",
|
| 17 |
+
"time_text_embed.timestep_embedder.linear_2",
|
| 18 |
+
"txt_norm",
|
| 19 |
+
"img_in",
|
| 20 |
+
"txt_in",
|
| 21 |
+
"transformer_blocks.*.img_mod.1",
|
| 22 |
+
"transformer_blocks.*.attn.norm_q",
|
| 23 |
+
"transformer_blocks.*.attn.norm_k",
|
| 24 |
+
"transformer_blocks.*.attn.to_q",
|
| 25 |
+
"transformer_blocks.*.attn.to_k",
|
| 26 |
+
"transformer_blocks.*.attn.to_v",
|
| 27 |
+
"transformer_blocks.*.attn.add_k_proj",
|
| 28 |
+
"transformer_blocks.*.attn.add_v_proj",
|
| 29 |
+
"transformer_blocks.*.attn.add_q_proj",
|
| 30 |
+
"transformer_blocks.*.attn.to_out.0",
|
| 31 |
+
"transformer_blocks.*.attn.to_add_out",
|
| 32 |
+
"transformer_blocks.*.attn.norm_added_q",
|
| 33 |
+
"transformer_blocks.*.attn.norm_added_k",
|
| 34 |
+
"transformer_blocks.*.img_mlp.net.0.proj",
|
| 35 |
+
"transformer_blocks.*.img_mlp.net.2",
|
| 36 |
+
"transformer_blocks.*.txt_mod.1",
|
| 37 |
+
"transformer_blocks.*.txt_mlp.net.0.proj",
|
| 38 |
+
"transformer_blocks.*.txt_mlp.net.2",
|
| 39 |
+
"norm_out.linear",
|
| 40 |
+
"proj_out",
|
| 41 |
+
]
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def convert_from_diffusers(prefix, weights_sd):
|
| 45 |
+
# convert from diffusers(?) to default LoRA
|
| 46 |
+
# Diffusers format: {"diffusion_model.module.name.lora_A.weight": weight, "diffusion_model.module.name.lora_B.weight": weight, ...}
|
| 47 |
+
# default LoRA format: {"prefix_module_name.lora_down.weight": weight, "prefix_module_name.lora_up.weight": weight, ...}
|
| 48 |
+
|
| 49 |
+
# note: Diffusers has no alpha, so alpha is set to rank
|
| 50 |
+
new_weights_sd = {}
|
| 51 |
+
lora_dims = {}
|
| 52 |
+
for key, weight in weights_sd.items():
|
| 53 |
+
diffusers_prefix, key_body = key.split(".", 1)
|
| 54 |
+
if diffusers_prefix != "diffusion_model" and diffusers_prefix != "transformer":
|
| 55 |
+
logger.warning(f"unexpected key: {key} in diffusers format")
|
| 56 |
+
continue
|
| 57 |
+
|
| 58 |
+
new_key = f"{prefix}{key_body}".replace(".", "_")
|
| 59 |
+
if "_lora_" in new_key: # LoRA
|
| 60 |
+
new_key = new_key.replace("_lora_A_", ".lora_down.").replace("_lora_B_", ".lora_up.")
|
| 61 |
+
|
| 62 |
+
# support unknown format: do not replace dots but uses lora_down/lora_up/alpha
|
| 63 |
+
new_key = new_key.replace("_lora_down_", ".lora_down.").replace("_lora_up_", ".lora_up.")
|
| 64 |
+
else: # LoHa or LoKr
|
| 65 |
+
new_key = new_key.replace("_hada_", ".hada_").replace("_lokr_", ".lokr_")
|
| 66 |
+
|
| 67 |
+
if new_key.endswith("_alpha"):
|
| 68 |
+
new_key = new_key.replace("_alpha", ".alpha")
|
| 69 |
+
|
| 70 |
+
new_weights_sd[new_key] = weight
|
| 71 |
+
|
| 72 |
+
lora_name = new_key.split(".")[0] # before first dot
|
| 73 |
+
if lora_name not in lora_dims and "lora_down" in new_key:
|
| 74 |
+
lora_dims[lora_name] = weight.shape[0]
|
| 75 |
+
|
| 76 |
+
# add alpha with rank
|
| 77 |
+
for lora_name, dim in lora_dims.items():
|
| 78 |
+
alpha_key = f"{lora_name}.alpha"
|
| 79 |
+
if alpha_key not in new_weights_sd:
|
| 80 |
+
new_weights_sd[f"{lora_name}.alpha"] = torch.tensor(dim)
|
| 81 |
+
|
| 82 |
+
return new_weights_sd
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def convert_to_diffusers(prefix, diffusers_prefix, weights_sd):
|
| 86 |
+
# convert from default LoRA to diffusers
|
| 87 |
+
if diffusers_prefix is None:
|
| 88 |
+
diffusers_prefix = "diffusion_model"
|
| 89 |
+
|
| 90 |
+
# make reverse map from LoRA name to base model module name
|
| 91 |
+
lora_name_to_module_name = {}
|
| 92 |
+
for key in QWEN_IMAGE_KEYS:
|
| 93 |
+
if "*" not in key:
|
| 94 |
+
lora_name = prefix + key.replace(".", "_")
|
| 95 |
+
lora_name_to_module_name[lora_name] = key
|
| 96 |
+
else:
|
| 97 |
+
lora_name = prefix + key.replace(".", "_")
|
| 98 |
+
for i in range(100): # assume at most 100 transformer blocks
|
| 99 |
+
lora_name_to_module_name[lora_name.replace("*", str(i))] = key.replace("*", str(i))
|
| 100 |
+
|
| 101 |
+
# get alphas
|
| 102 |
+
lora_alphas = {}
|
| 103 |
+
for key, weight in weights_sd.items():
|
| 104 |
+
if key.startswith(prefix):
|
| 105 |
+
lora_name = key.split(".", 1)[0] # before first dot
|
| 106 |
+
if lora_name not in lora_alphas and "alpha" in key:
|
| 107 |
+
lora_alphas[lora_name] = weight
|
| 108 |
+
|
| 109 |
+
new_weights_sd = {}
|
| 110 |
+
estimated_type = None
|
| 111 |
+
for key, weight in weights_sd.items():
|
| 112 |
+
if key.startswith(prefix):
|
| 113 |
+
if "alpha" in key:
|
| 114 |
+
continue
|
| 115 |
+
|
| 116 |
+
lora_name, weight_name = key.split(".", 1)
|
| 117 |
+
|
| 118 |
+
if lora_name in lora_name_to_module_name:
|
| 119 |
+
module_name = lora_name_to_module_name[lora_name]
|
| 120 |
+
else:
|
| 121 |
+
module_name = lora_name[len(prefix) :] # remove "lora_unet_"
|
| 122 |
+
module_name = module_name.replace("_", ".") # replace "_" with "."
|
| 123 |
+
if ".cross.attn." in module_name or ".self.attn." in module_name:
|
| 124 |
+
# Wan2.1 lora name to module name: ugly but works
|
| 125 |
+
module_name = module_name.replace("cross.attn", "cross_attn") # fix cross attn
|
| 126 |
+
module_name = module_name.replace("self.attn", "self_attn") # fix self attn
|
| 127 |
+
module_name = module_name.replace("k.img", "k_img") # fix k img
|
| 128 |
+
module_name = module_name.replace("v.img", "v_img") # fix v img
|
| 129 |
+
elif ".attention.to." in module_name or ".feed.forward." in module_name:
|
| 130 |
+
# Z-Image lora name to module name: ugly but works
|
| 131 |
+
module_name = module_name.replace("to.q", "to_q") # fix to q
|
| 132 |
+
module_name = module_name.replace("to.k", "to_k") # fix to k
|
| 133 |
+
module_name = module_name.replace("to.v", "to_v") # fix to v
|
| 134 |
+
module_name = module_name.replace("to.out", "to_out") # fix to out
|
| 135 |
+
module_name = module_name.replace("feed.forward", "feed_forward") # fix feed forward
|
| 136 |
+
elif "double.blocks." in module_name or "single.blocks." in module_name:
|
| 137 |
+
# HunyuanVideo and FLUX lora name to module name: ugly but works
|
| 138 |
+
module_name = module_name.replace("double.blocks.", "double_blocks.") # fix double blocks
|
| 139 |
+
module_name = module_name.replace("single.blocks.", "single_blocks.") # fix single blocks
|
| 140 |
+
module_name = module_name.replace("img.", "img_") # fix img
|
| 141 |
+
module_name = module_name.replace("txt.", "txt_") # fix txt
|
| 142 |
+
module_name = module_name.replace("attn.", "attn_") # fix attn
|
| 143 |
+
elif ".attn.qkv.proj" in module_name or ".attn.out.proj" in module_name or ".adaln.proj" in module_name:
|
| 144 |
+
# MiniMax-H3 lora name to module name: ugly but works
|
| 145 |
+
module_name = module_name.replace("qkv.proj", "qkv_proj") # fix qkv proj
|
| 146 |
+
module_name = module_name.replace("out.proj", "out_proj") # fix out proj
|
| 147 |
+
module_name = module_name.replace("adaln.proj", "adaln_proj") # fix adaln proj
|
| 148 |
+
|
| 149 |
+
dim = None # None means LoHa or LoKr, otherwise it's LoRA with alpha and dim is used for scaling
|
| 150 |
+
if "lora_down" in key:
|
| 151 |
+
new_key = f"{diffusers_prefix}.{module_name}.lora_A.weight"
|
| 152 |
+
dim = weight.shape[0]
|
| 153 |
+
elif "lora_up" in key:
|
| 154 |
+
new_key = f"{diffusers_prefix}.{module_name}.lora_B.weight"
|
| 155 |
+
dim = weight.shape[1]
|
| 156 |
+
elif "hada" in key or "lokr" in key: # LoHa or LoKr
|
| 157 |
+
new_key = f"{diffusers_prefix}.{module_name}.{weight_name}"
|
| 158 |
+
if "hada" in key:
|
| 159 |
+
estimated_type = "LoHa"
|
| 160 |
+
elif "lokr" in key:
|
| 161 |
+
estimated_type = "LoKr"
|
| 162 |
+
else:
|
| 163 |
+
logger.warning(f"unexpected key: {key} in default LoRA format")
|
| 164 |
+
continue
|
| 165 |
+
if dim is not None:
|
| 166 |
+
estimated_type = "LoRA"
|
| 167 |
+
|
| 168 |
+
# scale weight by alpha for LoRA with alpha (e.g., LyCORIS), to match Diffusers format which has no alpha (alpha is effectively 1)
|
| 169 |
+
if lora_name in lora_alphas and dim is not None:
|
| 170 |
+
# we scale both down and up, so scale is sqrt
|
| 171 |
+
scale = lora_alphas[lora_name] / dim
|
| 172 |
+
scale = scale.sqrt()
|
| 173 |
+
weight = weight * scale
|
| 174 |
+
else:
|
| 175 |
+
if dim is not None:
|
| 176 |
+
logger.warning(f"missing alpha for {lora_name}")
|
| 177 |
+
else:
|
| 178 |
+
# for LoHa or LoKr, we copy alpha if exists
|
| 179 |
+
if lora_name in lora_alphas:
|
| 180 |
+
new_weights_sd[f"{diffusers_prefix}.{module_name}.alpha"] = lora_alphas[lora_name]
|
| 181 |
+
|
| 182 |
+
new_weights_sd[new_key] = weight
|
| 183 |
+
|
| 184 |
+
logger.info(f"estimated type: {estimated_type}")
|
| 185 |
+
return new_weights_sd
|
| 186 |
+
|
| 187 |
+
|
| 188 |
+
def convert(input_file, output_file, target_format, diffusers_prefix):
|
| 189 |
+
logger.info(f"loading {input_file}")
|
| 190 |
+
weights_sd = load_file(input_file)
|
| 191 |
+
with safe_open(input_file, framework="pt") as f:
|
| 192 |
+
metadata = f.metadata()
|
| 193 |
+
|
| 194 |
+
logger.info(f"converting to {target_format}")
|
| 195 |
+
prefix = "lora_unet_"
|
| 196 |
+
if target_format == "default":
|
| 197 |
+
new_weights_sd = convert_from_diffusers(prefix, weights_sd)
|
| 198 |
+
metadata = metadata or {}
|
| 199 |
+
model_utils.precalculate_safetensors_hashes(new_weights_sd, metadata)
|
| 200 |
+
elif target_format == "other":
|
| 201 |
+
new_weights_sd = convert_to_diffusers(prefix, diffusers_prefix, weights_sd)
|
| 202 |
+
else:
|
| 203 |
+
raise ValueError(f"unknown target format: {target_format}")
|
| 204 |
+
|
| 205 |
+
logger.info(f"saving to {output_file}")
|
| 206 |
+
save_file(new_weights_sd, output_file, metadata=metadata)
|
| 207 |
+
|
| 208 |
+
logger.info("done")
|
| 209 |
+
|
| 210 |
+
|
| 211 |
+
def parse_args():
|
| 212 |
+
parser = argparse.ArgumentParser(description="Convert LoRA/LoHa/LoKr weights between default and other formats")
|
| 213 |
+
parser.add_argument("--input", type=str, required=True, help="input model file")
|
| 214 |
+
parser.add_argument("--output", type=str, required=True, help="output model file")
|
| 215 |
+
parser.add_argument("--target", type=str, required=True, choices=["other", "default"], help="target format")
|
| 216 |
+
parser.add_argument(
|
| 217 |
+
"--diffusers_prefix", type=str, default=None, help="prefix for Diffusers weights, default is None (use `diffusion_model`)"
|
| 218 |
+
)
|
| 219 |
+
args = parser.parse_args()
|
| 220 |
+
return args
|
| 221 |
+
|
| 222 |
+
|
| 223 |
+
def main():
|
| 224 |
+
args = parse_args()
|
| 225 |
+
convert(args.input, args.output, args.target, args.diffusers_prefix)
|
| 226 |
+
|
| 227 |
+
|
| 228 |
+
if __name__ == "__main__":
|
| 229 |
+
main()
|
third_party/musubi_tuner/dataset/__init__.py
ADDED
|
File without changes
|
third_party/musubi_tuner/dataset/audio_utils.py
ADDED
|
@@ -0,0 +1,512 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from dataclasses import dataclass, replace
|
| 4 |
+
import math
|
| 5 |
+
from pathlib import Path
|
| 6 |
+
from typing import Callable, Optional, Sequence
|
| 7 |
+
|
| 8 |
+
import av
|
| 9 |
+
import numpy as np
|
| 10 |
+
import torch
|
| 11 |
+
|
| 12 |
+
from musubi_tuner.dataset.media_utils import video_origin_seconds
|
| 13 |
+
|
| 14 |
+
import logging
|
| 15 |
+
|
| 16 |
+
logger = logging.getLogger(__name__)
|
| 17 |
+
|
| 18 |
+
AUDIO_SIDECAR_EXTENSIONS = frozenset({".aac", ".flac", ".m4a", ".mp3", ".ogg", ".opus", ".wav"})
|
| 19 |
+
|
| 20 |
+
# tolerance for timestamp gaps/overlaps between decoded audio chunks (codec jitter)
|
| 21 |
+
DEFAULT_TIMESTAMP_TOLERANCE_SAMPLES = 2
|
| 22 |
+
# The largest pts discontinuity between decoded chunks that is handled rather than an error.
|
| 23 |
+
# Chunk timestamps that wobble in both directions within this range around the decode-order
|
| 24 |
+
# sample positions come from a muxer stamping pts with a jittery clock (screen and USB captures
|
| 25 |
+
# on a 10 ms timer, re-encodes of broadcast cuts with 30 ms swings): the samples are contiguous
|
| 26 |
+
# and are concatenated in decode order. Larger or one-directional discontinuities (a cut at a
|
| 27 |
+
# non-frame boundary, capture stalls, an audio clock drifting against the muxer clock) are
|
| 28 |
+
# repaired in place so that the audio stays aligned to the video timestamps: a gap up to this
|
| 29 |
+
# size is zero-filled at its pts position and an overlap up to this size is trimmed. Within a
|
| 30 |
+
# wobble a one-off pts step cannot be told from the jitter; concatenation absorbs it as a shift
|
| 31 |
+
# of at most the range, below the threshold at which audio-visual misalignment is perceived.
|
| 32 |
+
DEFAULT_MAX_DISCONTINUITY_SECONDS = 0.05
|
| 33 |
+
# a training window whose missing audio (in-place repairs, silence before the audio starts or
|
| 34 |
+
# after it ends) adds up to more than this is rejected: that much is not worth teaching
|
| 35 |
+
DEFAULT_MAX_MISSING_SECONDS = 0.2
|
| 36 |
+
# tolerance for missing samples at the end of a stream (codec priming/padding); shortfalls
|
| 37 |
+
# within this tolerance are zero-padded without being recorded, larger shortfalls count as
|
| 38 |
+
# missing audio against the per-window limit
|
| 39 |
+
DEFAULT_CODEC_PAD_TOLERANCE_SAMPLES = 800
|
| 40 |
+
|
| 41 |
+
_CHANNEL_LAYOUTS = {1: "mono", 2: "stereo"}
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
@dataclass(frozen=True)
|
| 45 |
+
class AudioSource:
|
| 46 |
+
path: Path
|
| 47 |
+
embedded: bool # True if the audio is an audio track inside the video container
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
@dataclass(frozen=True)
|
| 51 |
+
class AudioSpec:
|
| 52 |
+
"""Architecture-specific audio dataset parameters.
|
| 53 |
+
|
| 54 |
+
Entry scripts of audio-capable architectures construct one and pass it to
|
| 55 |
+
`generate_dataset_group_by_blueprint`; the shared dataset layer stays
|
| 56 |
+
architecture-agnostic. `samples_per_crop` maps a video crop's frame count to the
|
| 57 |
+
number of waveform samples the architecture's audio latent grid requires.
|
| 58 |
+
|
| 59 |
+
`samples_per_crop` must be a picklable module-level function (not a lambda or
|
| 60 |
+
closure): datasets carry the spec into DataLoader workers, which are spawned
|
| 61 |
+
processes on Windows/macOS and pickle their arguments.
|
| 62 |
+
|
| 63 |
+
`max_discontinuity_seconds` and `max_missing_seconds` are the timeline tolerances of
|
| 64 |
+
`decode_audio` and `slice_audio_window`; cache scripts override them from the command
|
| 65 |
+
line (see `add_audio_tolerance_arguments`).
|
| 66 |
+
"""
|
| 67 |
+
|
| 68 |
+
sample_rate: int
|
| 69 |
+
channels: int
|
| 70 |
+
samples_per_crop: Callable[[int], int]
|
| 71 |
+
codec_pad_tolerance: int = DEFAULT_CODEC_PAD_TOLERANCE_SAMPLES
|
| 72 |
+
max_discontinuity_seconds: float = DEFAULT_MAX_DISCONTINUITY_SECONDS
|
| 73 |
+
max_missing_seconds: float = DEFAULT_MAX_MISSING_SECONDS
|
| 74 |
+
|
| 75 |
+
def __post_init__(self):
|
| 76 |
+
if self.sample_rate <= 0:
|
| 77 |
+
raise ValueError(f"Audio sample rate must be positive, got {self.sample_rate}")
|
| 78 |
+
if self.channels not in _CHANNEL_LAYOUTS:
|
| 79 |
+
raise ValueError(f"Audio channels must be one of {sorted(_CHANNEL_LAYOUTS)}, got {self.channels}")
|
| 80 |
+
if self.codec_pad_tolerance < 0:
|
| 81 |
+
raise ValueError(f"Audio codec pad tolerance must be nonnegative, got {self.codec_pad_tolerance}")
|
| 82 |
+
if self.max_discontinuity_seconds < 0 or self.max_missing_seconds < 0:
|
| 83 |
+
raise ValueError("Audio timeline tolerances must be nonnegative")
|
| 84 |
+
|
| 85 |
+
@property
|
| 86 |
+
def max_missing_samples(self) -> int:
|
| 87 |
+
"""The per-window cap on missing audio, in samples."""
|
| 88 |
+
return round(self.sample_rate * self.max_missing_seconds)
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
def add_audio_tolerance_arguments(parser) -> None:
|
| 92 |
+
"""Adds the audio timeline tolerance options to a cache script's argument parser."""
|
| 93 |
+
parser.add_argument(
|
| 94 |
+
"--audio_max_discontinuity",
|
| 95 |
+
type=float,
|
| 96 |
+
default=DEFAULT_MAX_DISCONTINUITY_SECONDS,
|
| 97 |
+
help="largest audio pts jump (seconds) handled by concatenation, silence fill or trimming; larger jumps fail."
|
| 98 |
+
f" Raise to force through badly muxed files at the cost of audio-visual alignment. Default {DEFAULT_MAX_DISCONTINUITY_SECONDS}",
|
| 99 |
+
)
|
| 100 |
+
parser.add_argument(
|
| 101 |
+
"--audio_max_missing",
|
| 102 |
+
type=float,
|
| 103 |
+
default=DEFAULT_MAX_MISSING_SECONDS,
|
| 104 |
+
help="most missing audio (seconds of silence fill, trimmed overlaps, audio starting late or ending early) allowed in"
|
| 105 |
+
f" one training window; windows with more fail. Default {DEFAULT_MAX_MISSING_SECONDS}",
|
| 106 |
+
)
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
def apply_audio_tolerance_arguments(spec: AudioSpec, args) -> AudioSpec:
|
| 110 |
+
"""Returns `spec` with the tolerances from `add_audio_tolerance_arguments` applied."""
|
| 111 |
+
return replace(spec, max_discontinuity_seconds=args.audio_max_discontinuity, max_missing_seconds=args.audio_max_missing)
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def probe_audio(path: str | Path) -> bool:
|
| 115 |
+
"""Returns True if the media file contains at least one audio stream."""
|
| 116 |
+
with av.open(str(path)) as container:
|
| 117 |
+
return bool(container.streams.audio)
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
def sidecar_audio_paths(video_path: str | Path) -> list[Path]:
|
| 121 |
+
"""Returns same-stem audio files next to the video, sorted by name."""
|
| 122 |
+
video_path = Path(video_path)
|
| 123 |
+
return sorted(
|
| 124 |
+
(
|
| 125 |
+
candidate.resolve()
|
| 126 |
+
for candidate in video_path.parent.iterdir()
|
| 127 |
+
if candidate.is_file() and candidate.stem == video_path.stem and candidate.suffix.lower() in AUDIO_SIDECAR_EXTENSIONS
|
| 128 |
+
),
|
| 129 |
+
key=lambda path: path.name.lower(),
|
| 130 |
+
)
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
def resolve_audio_source(video_path: str | Path, explicit_path: Optional[str | Path] = None) -> Optional[AudioSource]:
|
| 134 |
+
"""Resolves the audio source for a video item.
|
| 135 |
+
|
| 136 |
+
Priority: explicit path > same-stem sidecar file > audio track embedded in the video.
|
| 137 |
+
Returns None if the item has no audio. An explicit path that does not exist or has no
|
| 138 |
+
audio stream is an error; so is more than one sidecar candidate.
|
| 139 |
+
"""
|
| 140 |
+
video_path = Path(video_path).resolve()
|
| 141 |
+
|
| 142 |
+
if explicit_path is not None:
|
| 143 |
+
path = Path(explicit_path).resolve()
|
| 144 |
+
if not path.is_file():
|
| 145 |
+
raise ValueError(f"Explicit audio path does not exist: {path}")
|
| 146 |
+
if not probe_audio(path):
|
| 147 |
+
raise ValueError(f"Explicit audio source contains no audio stream: {path}")
|
| 148 |
+
return AudioSource(path=path, embedded=False)
|
| 149 |
+
|
| 150 |
+
sidecars = sidecar_audio_paths(video_path)
|
| 151 |
+
if len(sidecars) > 1:
|
| 152 |
+
formatted = ", ".join(str(path) for path in sidecars)
|
| 153 |
+
raise ValueError(f"Multiple same-stem audio sidecars found for {video_path}: {formatted}")
|
| 154 |
+
if sidecars:
|
| 155 |
+
if not probe_audio(sidecars[0]):
|
| 156 |
+
raise ValueError(f"Audio sidecar contains no audio stream: {sidecars[0]}")
|
| 157 |
+
return AudioSource(path=sidecars[0], embedded=False)
|
| 158 |
+
|
| 159 |
+
if video_path.is_file() and probe_audio(video_path):
|
| 160 |
+
return AudioSource(path=video_path, embedded=True)
|
| 161 |
+
return None
|
| 162 |
+
|
| 163 |
+
|
| 164 |
+
def _audio_frame_to_tensor(frame: av.AudioFrame, channels: int) -> torch.Tensor:
|
| 165 |
+
array = frame.to_ndarray()
|
| 166 |
+
if array.ndim != 2:
|
| 167 |
+
raise ValueError(f"Unexpected PyAV audio frame shape: {array.shape}")
|
| 168 |
+
if array.shape[0] != channels and array.shape[1] == channels:
|
| 169 |
+
array = array.T
|
| 170 |
+
if array.shape[0] != channels:
|
| 171 |
+
raise ValueError(f"PyAV resampler returned shape {array.shape}, expected {channels} channels")
|
| 172 |
+
return torch.from_numpy(np.asarray(array, dtype=np.float32).copy())
|
| 173 |
+
|
| 174 |
+
|
| 175 |
+
def _audio_frame_start_sample(frame: av.AudioFrame, sample_rate: int, fallback: int) -> int:
|
| 176 |
+
if frame.pts is None or frame.time_base is None:
|
| 177 |
+
return fallback
|
| 178 |
+
return round(frame.pts * frame.time_base * sample_rate)
|
| 179 |
+
|
| 180 |
+
|
| 181 |
+
@dataclass(frozen=True)
|
| 182 |
+
class AudioRepair:
|
| 183 |
+
"""One in-place timeline repair made while assembling decoded chunks.
|
| 184 |
+
|
| 185 |
+
`position` is the sample index in the assembled waveform where the repair sits; `filled`
|
| 186 |
+
zero samples were inserted for a pts gap (or, at position 0, for audio that starts after
|
| 187 |
+
the origin), `trimmed` samples were dropped for a pts overlap.
|
| 188 |
+
"""
|
| 189 |
+
|
| 190 |
+
position: int
|
| 191 |
+
filled: int = 0
|
| 192 |
+
trimmed: int = 0
|
| 193 |
+
|
| 194 |
+
|
| 195 |
+
@dataclass(frozen=True)
|
| 196 |
+
class DecodedAudio:
|
| 197 |
+
"""A [C, L] waveform plus the timeline repairs that went into it."""
|
| 198 |
+
|
| 199 |
+
waveform: torch.Tensor
|
| 200 |
+
repairs: tuple[AudioRepair, ...] = ()
|
| 201 |
+
|
| 202 |
+
def repaired_samples(self, start_sample: int, sample_count: int) -> int:
|
| 203 |
+
"""Total repaired samples whose position falls inside the window."""
|
| 204 |
+
return repaired_samples_in_window(self.repairs, start_sample, sample_count)
|
| 205 |
+
|
| 206 |
+
|
| 207 |
+
def repaired_samples_in_window(repairs: Sequence[AudioRepair], start_sample: int, sample_count: int) -> int:
|
| 208 |
+
end = start_sample + sample_count
|
| 209 |
+
return sum(repair.filled + repair.trimmed for repair in repairs if start_sample <= repair.position < end)
|
| 210 |
+
|
| 211 |
+
|
| 212 |
+
def max_missing_samples(sample_rate: int, max_missing_seconds: float = DEFAULT_MAX_MISSING_SECONDS) -> int:
|
| 213 |
+
"""The per-window cap on missing audio for a stream at `sample_rate`, in samples."""
|
| 214 |
+
return round(sample_rate * max_missing_seconds)
|
| 215 |
+
|
| 216 |
+
|
| 217 |
+
def _is_pts_jitter(deviations: Sequence[int], jitter_range: int, tolerance: int) -> bool:
|
| 218 |
+
"""True if chunk timestamps merely wobble around the decode-order positions.
|
| 219 |
+
|
| 220 |
+
Jitter is bounded (the deviations span at most `jitter_range`) and goes both ways: the
|
| 221 |
+
per-chunk steps include at least one gap and at least one overlap beyond the tolerance. A
|
| 222 |
+
one-directional series is a drift or a cut, whose samples really are missing or duplicated.
|
| 223 |
+
"""
|
| 224 |
+
if max(deviations) - min(deviations) > jitter_range:
|
| 225 |
+
return False
|
| 226 |
+
steps = [later - earlier for earlier, later in zip(deviations, deviations[1:])]
|
| 227 |
+
return any(step > tolerance for step in steps) and any(step < -tolerance for step in steps)
|
| 228 |
+
|
| 229 |
+
|
| 230 |
+
def _place_at_origin(decoded: DecodedAudio, offset_samples: int, tolerance: int, context_suffix: str) -> DecodedAudio:
|
| 231 |
+
"""Moves the waveform's origin by `offset_samples`, the first chunk's start relative to the origin.
|
| 232 |
+
|
| 233 |
+
A positive offset means the audio starts after the origin: silence is inserted in front and,
|
| 234 |
+
beyond `tolerance`, recorded as a repair at position 0. A negative offset means it starts
|
| 235 |
+
before: those samples are dropped (nothing on the origin's timeline is lost, so this is not
|
| 236 |
+
a repair) and the recorded repairs move with the waveform.
|
| 237 |
+
"""
|
| 238 |
+
if offset_samples == 0:
|
| 239 |
+
return decoded
|
| 240 |
+
waveform = decoded.waveform
|
| 241 |
+
if offset_samples > 0:
|
| 242 |
+
silence = torch.zeros(waveform.shape[0], offset_samples, dtype=waveform.dtype, device=waveform.device)
|
| 243 |
+
repairs = [AudioRepair(0, filled=offset_samples)] if offset_samples > tolerance else []
|
| 244 |
+
repairs.extend(AudioRepair(repair.position + offset_samples, repair.filled, repair.trimmed) for repair in decoded.repairs)
|
| 245 |
+
return DecodedAudio(torch.cat([silence, waveform], dim=1), tuple(repairs))
|
| 246 |
+
|
| 247 |
+
dropped = -offset_samples
|
| 248 |
+
if dropped >= waveform.shape[1]:
|
| 249 |
+
raise ValueError(f"Audio stream ends before the first video frame{context_suffix}")
|
| 250 |
+
repairs = []
|
| 251 |
+
for repair in decoded.repairs:
|
| 252 |
+
position = repair.position - dropped
|
| 253 |
+
if position >= 0:
|
| 254 |
+
repairs.append(AudioRepair(position, repair.filled, repair.trimmed))
|
| 255 |
+
elif repair.filled > -position: # a gap straddling the origin: keep the part inside
|
| 256 |
+
repairs.append(AudioRepair(0, filled=repair.filled + position))
|
| 257 |
+
return DecodedAudio(waveform[:, dropped:], tuple(repairs))
|
| 258 |
+
|
| 259 |
+
|
| 260 |
+
def assemble_audio_chunks(
|
| 261 |
+
chunks: Sequence[tuple[int, torch.Tensor]],
|
| 262 |
+
*,
|
| 263 |
+
channels: int,
|
| 264 |
+
timestamp_tolerance_samples: int = DEFAULT_TIMESTAMP_TOLERANCE_SAMPLES,
|
| 265 |
+
pts_jitter_range_samples: int = 0,
|
| 266 |
+
max_gap_fill_samples: int = 0,
|
| 267 |
+
max_overlap_trim_samples: int = 0,
|
| 268 |
+
origin_sample: Optional[int] = None,
|
| 269 |
+
sample_rate: Optional[int] = None,
|
| 270 |
+
context: str = "",
|
| 271 |
+
) -> DecodedAudio:
|
| 272 |
+
"""Concatenates (start_sample, [C, L]) chunks into one waveform.
|
| 273 |
+
|
| 274 |
+
Timestamps that only wobble around the decode-order sample positions (see `_is_pts_jitter`)
|
| 275 |
+
carry no missing or duplicated samples: the stream is contiguous and merely stamped with a
|
| 276 |
+
jittery clock, so the chunks are concatenated in decode order, ignoring pts.
|
| 277 |
+
`pts_jitter_range_samples` bounds the wobble; 0 disables this path.
|
| 278 |
+
|
| 279 |
+
Otherwise every chunk is placed at its pts position: a gap up to `max_gap_fill_samples` is
|
| 280 |
+
zero-filled and an overlap up to `max_overlap_trim_samples` is trimmed, each recorded as an
|
| 281 |
+
`AudioRepair` (gaps and overlaps within `timestamp_tolerance_samples` are handled the same
|
| 282 |
+
way but not recorded). A larger discontinuity is an error.
|
| 283 |
+
|
| 284 |
+
The waveform's sample 0 sits at `origin_sample` (the first chunk's start when None): audio
|
| 285 |
+
before the origin is dropped, audio starting after it is preceded by silence, recorded as a
|
| 286 |
+
repair at position 0 when it exceeds the timestamp tolerance (see `_place_at_origin`). `sample_rate` and `context` only make the
|
| 287 |
+
error messages readable.
|
| 288 |
+
"""
|
| 289 |
+
if not chunks:
|
| 290 |
+
raise ValueError("Audio chunk list is empty")
|
| 291 |
+
if timestamp_tolerance_samples < 0:
|
| 292 |
+
raise ValueError("Audio timestamp tolerance must be nonnegative")
|
| 293 |
+
if pts_jitter_range_samples < 0 or max_gap_fill_samples < 0 or max_overlap_trim_samples < 0:
|
| 294 |
+
raise ValueError("Audio pts jitter range and repair limits must be nonnegative")
|
| 295 |
+
for _, chunk in chunks:
|
| 296 |
+
if chunk.ndim != 2 or chunk.shape[0] != channels:
|
| 297 |
+
raise ValueError(f"Audio chunk must be [{channels},L], got {tuple(chunk.shape)}")
|
| 298 |
+
|
| 299 |
+
suffix = f": {context}" if context else ""
|
| 300 |
+
origin_offset = 0 if origin_sample is None else chunks[0][0] - origin_sample
|
| 301 |
+
|
| 302 |
+
if pts_jitter_range_samples > 0:
|
| 303 |
+
deviations = []
|
| 304 |
+
nominal_start = chunks[0][0]
|
| 305 |
+
for start_sample, chunk in chunks:
|
| 306 |
+
deviations.append(start_sample - nominal_start)
|
| 307 |
+
nominal_start += chunk.shape[1]
|
| 308 |
+
if _is_pts_jitter(deviations, pts_jitter_range_samples, timestamp_tolerance_samples):
|
| 309 |
+
logger.debug(
|
| 310 |
+
f"Audio pts jitter spanning {max(deviations) - min(deviations)} samples; concatenating chunks in decode order"
|
| 311 |
+
)
|
| 312 |
+
return _place_at_origin(
|
| 313 |
+
DecodedAudio(torch.cat([chunk for _, chunk in chunks], dim=1)), origin_offset, timestamp_tolerance_samples, suffix
|
| 314 |
+
)
|
| 315 |
+
|
| 316 |
+
def describe(samples: int, signed: bool = True) -> str:
|
| 317 |
+
sign = "+" if signed else ""
|
| 318 |
+
if sample_rate is None:
|
| 319 |
+
return f"{samples:{sign}d} samples"
|
| 320 |
+
return f"{samples * 1000 / sample_rate:{sign}.1f} ms ({samples:{sign}d} samples)"
|
| 321 |
+
|
| 322 |
+
expected_start = chunks[0][0]
|
| 323 |
+
assembled = []
|
| 324 |
+
repairs = []
|
| 325 |
+
position = 0
|
| 326 |
+
for start_sample, chunk in chunks:
|
| 327 |
+
delta = start_sample - expected_start
|
| 328 |
+
if abs(delta) > timestamp_tolerance_samples:
|
| 329 |
+
limit = max_gap_fill_samples if delta > 0 else max_overlap_trim_samples
|
| 330 |
+
if abs(delta) > limit:
|
| 331 |
+
where = f"sample {position}" if sample_rate is None else f"{position / sample_rate:.3f}s"
|
| 332 |
+
raise ValueError(
|
| 333 |
+
f"Audio stream is discontinuous at {where}: pts jumps by {describe(delta)}, beyond the repairable"
|
| 334 |
+
f" {describe(limit, signed=False)}{suffix}. Re-encode the audio so its timestamps are contiguous"
|
| 335 |
+
" (e.g. ffmpeg -c:v copy -af aresample=async=1:min_hard_comp=0.001 -c:a aac) or drop the audio."
|
| 336 |
+
)
|
| 337 |
+
if delta > 0:
|
| 338 |
+
repairs.append(AudioRepair(position, filled=delta))
|
| 339 |
+
else:
|
| 340 |
+
repairs.append(AudioRepair(position, trimmed=min(-delta, chunk.shape[1])))
|
| 341 |
+
if delta > 0:
|
| 342 |
+
assembled.append(torch.zeros(channels, delta, dtype=chunk.dtype, device=chunk.device))
|
| 343 |
+
position += delta
|
| 344 |
+
elif delta < 0:
|
| 345 |
+
chunk = chunk[:, -delta:]
|
| 346 |
+
if chunk.shape[1] > 0:
|
| 347 |
+
assembled.append(chunk)
|
| 348 |
+
position += chunk.shape[1]
|
| 349 |
+
expected_start = start_sample + chunk.shape[1] + max(0, -delta)
|
| 350 |
+
return _place_at_origin(
|
| 351 |
+
DecodedAudio(torch.cat(assembled, dim=1), tuple(repairs)), origin_offset, timestamp_tolerance_samples, suffix
|
| 352 |
+
)
|
| 353 |
+
|
| 354 |
+
|
| 355 |
+
def decode_audio(
|
| 356 |
+
source: AudioSource,
|
| 357 |
+
*,
|
| 358 |
+
sample_rate: int,
|
| 359 |
+
channels: int,
|
| 360 |
+
max_discontinuity_seconds: float = DEFAULT_MAX_DISCONTINUITY_SECONDS,
|
| 361 |
+
) -> DecodedAudio:
|
| 362 |
+
"""Decodes `source` to a [C, L] waveform at `sample_rate`, honoring the stream's timestamps.
|
| 363 |
+
|
| 364 |
+
`max_discontinuity_seconds` bounds the pts discontinuities that are handled (see
|
| 365 |
+
`DEFAULT_MAX_DISCONTINUITY_SECONDS`); a larger jump is an error naming the position.
|
| 366 |
+
"""
|
| 367 |
+
"""Decodes the full audio stream to a [channels, samples] float32 waveform.
|
| 368 |
+
|
| 369 |
+
Embedded audio is placed on the video's timeline: sample 0 is the first video frame's
|
| 370 |
+
timestamp (the origin of the frame grid `load_video` builds in its timestamps mode), so
|
| 371 |
+
audio recorded before the first frame is dropped and audio that starts after it is preceded
|
| 372 |
+
by silence. Muxers routinely start the two streams at different times (seconds apart on USB
|
| 373 |
+
captures, 100-200 ms on screen recordings). A sidecar or explicit audio file has no shared
|
| 374 |
+
clock with the video and starts at its own first sample.
|
| 375 |
+
|
| 376 |
+
Timeline defects are handled as `assemble_audio_chunks` describes; repairs beyond the
|
| 377 |
+
timestamp tolerance are logged per file and returned for the per-window check in
|
| 378 |
+
`slice_audio_window`.
|
| 379 |
+
"""
|
| 380 |
+
layout = _CHANNEL_LAYOUTS.get(channels)
|
| 381 |
+
if layout is None:
|
| 382 |
+
raise ValueError(f"Audio channels must be one of {sorted(_CHANNEL_LAYOUTS)}, got {channels}")
|
| 383 |
+
|
| 384 |
+
chunks = []
|
| 385 |
+
next_start_sample = 0
|
| 386 |
+
with av.open(str(source.path)) as container:
|
| 387 |
+
if not container.streams.audio:
|
| 388 |
+
raise ValueError(f"Audio source has no audio stream: {source.path}")
|
| 389 |
+
stream = container.streams.audio[0]
|
| 390 |
+
# containers with a coarse timestamp grid (e.g. Matroska's 1 ms time base) quantize
|
| 391 |
+
# chunk timestamps; two consecutive chunks can be quantized in opposite directions,
|
| 392 |
+
# so allow up to one full tick of jitter when assembling chunks
|
| 393 |
+
timestamp_tolerance = DEFAULT_TIMESTAMP_TOLERANCE_SAMPLES
|
| 394 |
+
if stream.time_base is not None:
|
| 395 |
+
tick_samples = float(stream.time_base) * sample_rate
|
| 396 |
+
timestamp_tolerance = max(timestamp_tolerance, math.ceil(tick_samples) + 1)
|
| 397 |
+
resampler = av.AudioResampler(format="fltp", layout=layout, rate=sample_rate)
|
| 398 |
+
for frame in container.decode(stream):
|
| 399 |
+
for resampled in resampler.resample(frame):
|
| 400 |
+
chunk = _audio_frame_to_tensor(resampled, channels)
|
| 401 |
+
chunk_start = _audio_frame_start_sample(resampled, sample_rate, next_start_sample)
|
| 402 |
+
chunks.append((chunk_start, chunk))
|
| 403 |
+
next_start_sample = chunk_start + chunk.shape[1]
|
| 404 |
+
for resampled in resampler.resample(None):
|
| 405 |
+
chunk = _audio_frame_to_tensor(resampled, channels)
|
| 406 |
+
chunk_start = _audio_frame_start_sample(resampled, sample_rate, next_start_sample)
|
| 407 |
+
chunks.append((chunk_start, chunk))
|
| 408 |
+
next_start_sample = chunk_start + chunk.shape[1]
|
| 409 |
+
|
| 410 |
+
if not chunks:
|
| 411 |
+
raise ValueError(f"Audio source decoded no samples: {source.path}")
|
| 412 |
+
|
| 413 |
+
origin_sample = None
|
| 414 |
+
if source.embedded:
|
| 415 |
+
origin_seconds = video_origin_seconds(str(source.path))
|
| 416 |
+
if origin_seconds is not None:
|
| 417 |
+
origin_sample = round(origin_seconds * sample_rate)
|
| 418 |
+
|
| 419 |
+
max_discontinuity = round(sample_rate * max_discontinuity_seconds)
|
| 420 |
+
decoded = assemble_audio_chunks(
|
| 421 |
+
chunks,
|
| 422 |
+
channels=channels,
|
| 423 |
+
timestamp_tolerance_samples=timestamp_tolerance,
|
| 424 |
+
pts_jitter_range_samples=max_discontinuity,
|
| 425 |
+
max_gap_fill_samples=max_discontinuity,
|
| 426 |
+
max_overlap_trim_samples=max_discontinuity,
|
| 427 |
+
origin_sample=origin_sample,
|
| 428 |
+
sample_rate=sample_rate,
|
| 429 |
+
context=str(source.path),
|
| 430 |
+
)
|
| 431 |
+
repairs = decoded.repairs
|
| 432 |
+
if origin_sample is not None:
|
| 433 |
+
offset = chunks[0][0] - origin_sample
|
| 434 |
+
if offset > timestamp_tolerance:
|
| 435 |
+
repairs = repairs[1:] # the lead silence is reported here, not as a discontinuity
|
| 436 |
+
if abs(offset) > max_discontinuity:
|
| 437 |
+
relation = "after" if offset > 0 else "before"
|
| 438 |
+
remedy = "silence fills the lead" if offset > 0 else "the lead is dropped"
|
| 439 |
+
logger.info(
|
| 440 |
+
f"Audio starts {abs(offset) * 1000 / sample_rate:.1f} ms {relation} the first video frame in {source.path};"
|
| 441 |
+
f" {remedy} to keep it aligned"
|
| 442 |
+
)
|
| 443 |
+
if repairs:
|
| 444 |
+
gaps = [repair.filled for repair in repairs if repair.filled]
|
| 445 |
+
overlaps = [repair.trimmed for repair in repairs if repair.trimmed]
|
| 446 |
+
to_ms = 1000 / sample_rate
|
| 447 |
+
logger.warning(
|
| 448 |
+
f"Audio pts discontinuities repaired in {source.path}: {len(gaps)} gaps zero-filled"
|
| 449 |
+
f" ({sum(gaps) * to_ms:.1f} ms total, largest {max(gaps, default=0) * to_ms:.1f} ms),"
|
| 450 |
+
f" {len(overlaps)} overlaps trimmed ({sum(overlaps) * to_ms:.1f} ms total)"
|
| 451 |
+
)
|
| 452 |
+
return DecodedAudio(decoded.waveform.contiguous(), decoded.repairs)
|
| 453 |
+
|
| 454 |
+
|
| 455 |
+
def slice_audio_window(
|
| 456 |
+
waveform: torch.Tensor,
|
| 457 |
+
*,
|
| 458 |
+
start_sample: int,
|
| 459 |
+
sample_count: int,
|
| 460 |
+
pad_tolerance: int = DEFAULT_CODEC_PAD_TOLERANCE_SAMPLES,
|
| 461 |
+
require_exact: bool = True,
|
| 462 |
+
context: str = "",
|
| 463 |
+
repairs: Sequence[AudioRepair] = (),
|
| 464 |
+
max_repair_samples: Optional[int] = None,
|
| 465 |
+
) -> torch.Tensor:
|
| 466 |
+
"""Extracts [C, sample_count] from a [C, L] waveform.
|
| 467 |
+
|
| 468 |
+
A terminal shortfall within pad_tolerance is zero-padded (codec priming/padding). When the
|
| 469 |
+
waveform's timeline `repairs` are given with `max_repair_samples`, the window's missing
|
| 470 |
+
audio (repaired samples inside it plus a larger terminal shortfall, i.e. video running past
|
| 471 |
+
the end of the audio) is padded with silence up to that budget and is an error beyond it:
|
| 472 |
+
the audio inside such a window is too broken or too absent to teach. Without a budget a
|
| 473 |
+
shortfall beyond pad_tolerance is an error when require_exact is True.
|
| 474 |
+
"""
|
| 475 |
+
if waveform.ndim != 2:
|
| 476 |
+
raise ValueError(f"Audio waveform must be [C, L], got {tuple(waveform.shape)}")
|
| 477 |
+
if start_sample < 0 or sample_count <= 0:
|
| 478 |
+
raise ValueError("Audio window must have a nonnegative start and positive length")
|
| 479 |
+
|
| 480 |
+
suffix = f": {context}" if context else ""
|
| 481 |
+
repaired = repaired_samples_in_window(repairs, start_sample, sample_count) if repairs else 0
|
| 482 |
+
if max_repair_samples is not None and repaired > max_repair_samples:
|
| 483 |
+
raise ValueError(
|
| 484 |
+
f"Audio window at sample {start_sample} contains {repaired} repaired samples"
|
| 485 |
+
f" (zero-filled gaps and trimmed overlaps), beyond the {max_repair_samples} allowed{suffix}"
|
| 486 |
+
)
|
| 487 |
+
window = waveform[:, start_sample : start_sample + sample_count]
|
| 488 |
+
if require_exact and window.shape[1] < sample_count:
|
| 489 |
+
deficit = sample_count - window.shape[1]
|
| 490 |
+
if deficit > pad_tolerance:
|
| 491 |
+
if max_repair_samples is None or repaired + deficit > max_repair_samples:
|
| 492 |
+
budget = "" if max_repair_samples is None else f" plus {repaired} repaired, beyond the {max_repair_samples} allowed"
|
| 493 |
+
raise ValueError(
|
| 494 |
+
f"Audio source is materially short at sample {start_sample}: need {sample_count}, got {window.shape[1]}"
|
| 495 |
+
f" ({deficit} missing samples at the end{budget}){suffix}"
|
| 496 |
+
)
|
| 497 |
+
logger.warning(
|
| 498 |
+
f"Audio ends {deficit} samples before the window at sample {start_sample} does; the rest is silence{suffix}"
|
| 499 |
+
)
|
| 500 |
+
window = torch.nn.functional.pad(window, (0, deficit))
|
| 501 |
+
if window.shape[1] == 0:
|
| 502 |
+
raise ValueError(f"Audio window is empty at sample {start_sample}{suffix}")
|
| 503 |
+
return window.contiguous()
|
| 504 |
+
|
| 505 |
+
|
| 506 |
+
def audio_window_start(crop_start_frame: int, fps: int, sample_rate: int) -> int:
|
| 507 |
+
"""Maps a video crop start frame (at integer fps) to the nearest waveform sample."""
|
| 508 |
+
if crop_start_frame < 0:
|
| 509 |
+
raise ValueError(f"Crop start frame must be nonnegative, got {crop_start_frame}")
|
| 510 |
+
if fps <= 0 or sample_rate <= 0:
|
| 511 |
+
raise ValueError("fps and sample_rate must be positive")
|
| 512 |
+
return (crop_start_frame * sample_rate + fps // 2) // fps
|
third_party/musubi_tuner/dataset/datasources.py
ADDED
|
@@ -0,0 +1,867 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from dataclasses import dataclass
|
| 4 |
+
import json
|
| 5 |
+
import os
|
| 6 |
+
from typing import Any, Mapping, Optional, TYPE_CHECKING
|
| 7 |
+
|
| 8 |
+
from PIL import Image
|
| 9 |
+
|
| 10 |
+
from musubi_tuner.dataset.audio_utils import AudioSource, AudioSpec, DecodedAudio, decode_audio, resolve_audio_source
|
| 11 |
+
from musubi_tuner.dataset.media_utils import glob_images, glob_videos, load_video, VIDEO_EXTENSIONS
|
| 12 |
+
|
| 13 |
+
if TYPE_CHECKING:
|
| 14 |
+
from musubi_tuner.dataset.bucket import BucketSelector
|
| 15 |
+
|
| 16 |
+
import logging
|
| 17 |
+
|
| 18 |
+
logger = logging.getLogger(__name__)
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
@dataclass(frozen=True)
|
| 22 |
+
class ItemExtras:
|
| 23 |
+
"""Per-item fields beyond the shared dataset schema, for architecture-specific consumers.
|
| 24 |
+
|
| 25 |
+
``fields`` holds the record's keys that the shared schema does not define (for example the
|
| 26 |
+
MiniMax-H3 ``references`` and ``teacher_caption``). Only record-based datasources (JSONL)
|
| 27 |
+
can carry such fields; directory datasources always report an empty mapping. Relative
|
| 28 |
+
paths inside ``fields`` resolve from ``base_directory`` (the JSONL's directory, or the
|
| 29 |
+
dataset directory), and ``label`` names the record's origin for error messages.
|
| 30 |
+
"""
|
| 31 |
+
|
| 32 |
+
fields: Mapping[str, Any]
|
| 33 |
+
base_directory: str
|
| 34 |
+
label: str
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def _extra_fields(data: Mapping[str, Any], shared_keys: tuple[str, ...]) -> dict[str, Any]:
|
| 38 |
+
"""The record's keys outside the shared schema; ``key_N`` counts as its ``key`` (control_path_0, image_path_1, ...)."""
|
| 39 |
+
extras = {}
|
| 40 |
+
for key, value in data.items():
|
| 41 |
+
stem, _, suffix = key.rpartition("_")
|
| 42 |
+
if key in shared_keys or (suffix.isdigit() and stem in shared_keys):
|
| 43 |
+
continue
|
| 44 |
+
extras[key] = value
|
| 45 |
+
return extras
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
class ContentDatasource:
|
| 49 |
+
def __init__(self):
|
| 50 |
+
self.caption_only = False # set to True to only fetch caption for Text Encoder caching
|
| 51 |
+
self.has_control = False
|
| 52 |
+
|
| 53 |
+
def set_caption_only(self, caption_only: bool):
|
| 54 |
+
self.caption_only = caption_only
|
| 55 |
+
|
| 56 |
+
def is_indexable(self):
|
| 57 |
+
return False
|
| 58 |
+
|
| 59 |
+
def get_caption(self, idx: int) -> tuple[str, str]:
|
| 60 |
+
"""
|
| 61 |
+
Returns caption. May not be called if is_indexable() returns False.
|
| 62 |
+
"""
|
| 63 |
+
raise NotImplementedError
|
| 64 |
+
|
| 65 |
+
def get_item_extras(self, idx: int) -> ItemExtras:
|
| 66 |
+
"""
|
| 67 |
+
Returns the item's fields beyond the shared schema (see ItemExtras). Indices align with
|
| 68 |
+
get_caption and with ItemInfo.datasource_index. May not be called if is_indexable() returns False.
|
| 69 |
+
"""
|
| 70 |
+
raise NotImplementedError
|
| 71 |
+
|
| 72 |
+
def __len__(self):
|
| 73 |
+
raise NotImplementedError
|
| 74 |
+
|
| 75 |
+
def __iter__(self):
|
| 76 |
+
raise NotImplementedError
|
| 77 |
+
|
| 78 |
+
def __next__(self):
|
| 79 |
+
raise NotImplementedError
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
class ImageDatasource(ContentDatasource):
|
| 83 |
+
def __init__(self):
|
| 84 |
+
super().__init__()
|
| 85 |
+
|
| 86 |
+
def get_image_data(self, idx: int) -> tuple[str, list[Image.Image], str, list[Image.Image]]:
|
| 87 |
+
"""
|
| 88 |
+
Returns image data as a tuple of image path, image, and caption for the given index.
|
| 89 |
+
Key must be unique and valid as a file name.
|
| 90 |
+
May not be called if is_indexable() returns False.
|
| 91 |
+
"""
|
| 92 |
+
raise NotImplementedError
|
| 93 |
+
|
| 94 |
+
def get_control_paths(self) -> dict[str, list[str]]:
|
| 95 |
+
"""
|
| 96 |
+
Returns {image_path: [control paths in index order]} for cache fingerprinting.
|
| 97 |
+
Control pixels travel through ItemInfo.control_content; this accessor exists only so
|
| 98 |
+
cache scripts can fingerprint the source files behind them.
|
| 99 |
+
"""
|
| 100 |
+
return {}
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
def _create_image_fetcher(datasource: ImageDatasource, index: int):
|
| 104 |
+
def fetch():
|
| 105 |
+
return datasource.get_image_data(index)
|
| 106 |
+
|
| 107 |
+
# the datasource record index travels as a fetcher attribute so that ItemInfo can
|
| 108 |
+
# reference the originating record without re-deriving it from item keys
|
| 109 |
+
fetch.datasource_index = index
|
| 110 |
+
return fetch
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
class ImageDirectoryDatasource(ImageDatasource):
|
| 114 |
+
def __init__(
|
| 115 |
+
self,
|
| 116 |
+
image_directory: str,
|
| 117 |
+
caption_extension: Optional[str] = None,
|
| 118 |
+
control_directory: Optional[str] = None,
|
| 119 |
+
control_count_per_image: Optional[int] = None,
|
| 120 |
+
multiple_target: bool = False,
|
| 121 |
+
):
|
| 122 |
+
super().__init__()
|
| 123 |
+
self.image_directory = image_directory
|
| 124 |
+
self.caption_extension = caption_extension
|
| 125 |
+
self.control_directory = control_directory
|
| 126 |
+
self.control_count_per_image = control_count_per_image
|
| 127 |
+
self.multiple_target = multiple_target
|
| 128 |
+
self.current_idx = 0
|
| 129 |
+
|
| 130 |
+
# glob images
|
| 131 |
+
logger.info(f"glob images in {self.image_directory}")
|
| 132 |
+
self.image_paths = glob_images(self.image_directory, caption_extension=self.caption_extension)
|
| 133 |
+
logger.info(f"found {len(self.image_paths)} images")
|
| 134 |
+
|
| 135 |
+
# check if multiple-target images exist
|
| 136 |
+
self.target_paths: dict[str, list[str]] = {} # image_path -> list of target image paths
|
| 137 |
+
|
| 138 |
+
if self.multiple_target:
|
| 139 |
+
# sort by length, longer first
|
| 140 |
+
sorted_image_paths = sorted(self.image_paths, key=lambda p: len(os.path.basename(p)), reverse=True)
|
| 141 |
+
|
| 142 |
+
all_image_paths = set(glob_images(self.image_directory)) # image1.jpg, image1_1.jpg, image1_2.jpg, ...
|
| 143 |
+
multiple_target_candidates = all_image_paths - set(sorted_image_paths) # those not in the images with captions
|
| 144 |
+
|
| 145 |
+
if len(multiple_target_candidates) > 0:
|
| 146 |
+
logger.info("checking for multiple-target images")
|
| 147 |
+
for image_path in sorted_image_paths:
|
| 148 |
+
image_path_no_ext = os.path.splitext(image_path)[0]
|
| 149 |
+
|
| 150 |
+
# find matching multiple-target images
|
| 151 |
+
potential_paths = [p for p in multiple_target_candidates if p.startswith(image_path_no_ext + "_")]
|
| 152 |
+
|
| 153 |
+
if potential_paths:
|
| 154 |
+
# sort by the digits (`_0000`) suffix
|
| 155 |
+
def sort_key(path):
|
| 156 |
+
path_no_ext = os.path.splitext(path)[0]
|
| 157 |
+
digits_suffix = path_no_ext.rsplit("_", 1)[-1]
|
| 158 |
+
if not digits_suffix.isdigit():
|
| 159 |
+
raise ValueError(
|
| 160 |
+
f"Invalid digits suffix in '{path_no_ext}'. Expected a numeric suffix after '_' "
|
| 161 |
+
f"(e.g., '_0', '_1', '_2') for proper sorting of multiple target images."
|
| 162 |
+
)
|
| 163 |
+
return int(digits_suffix)
|
| 164 |
+
|
| 165 |
+
potential_paths.sort(key=sort_key)
|
| 166 |
+
self.target_paths[image_path] = potential_paths
|
| 167 |
+
|
| 168 |
+
# remove to avoid duplicate matching
|
| 169 |
+
multiple_target_candidates.difference_update(potential_paths)
|
| 170 |
+
|
| 171 |
+
# check the number of targets: all multiple-target images should have the same number of targets
|
| 172 |
+
num_targets = 0
|
| 173 |
+
for image_path, paths in self.target_paths.items():
|
| 174 |
+
if num_targets == 0:
|
| 175 |
+
num_targets = len(paths)
|
| 176 |
+
elif num_targets != len(paths):
|
| 177 |
+
logger.error(
|
| 178 |
+
f"All multiple-target images must have the same number of targets / 全ての複数ターゲット画像は同じ数のターゲットを持つ必要があります: {image_path}"
|
| 179 |
+
)
|
| 180 |
+
raise ValueError(
|
| 181 |
+
f"All multiple-target images must have the same number of targets / 全ての複数ターゲット画像は同じ数のターゲットを持つ必要があります: {image_path}"
|
| 182 |
+
)
|
| 183 |
+
|
| 184 |
+
if num_targets == 0:
|
| 185 |
+
logger.error("no multiple-target images found, but multiple_target is set to True")
|
| 186 |
+
raise ValueError("no multiple-target images found, but multiple_target is set to True")
|
| 187 |
+
|
| 188 |
+
logger.info(f"found multiple-target images, max targets per image: {num_targets}")
|
| 189 |
+
|
| 190 |
+
# glob control images if specified
|
| 191 |
+
if self.control_directory is not None:
|
| 192 |
+
logger.info(f"glob control images in {self.control_directory}")
|
| 193 |
+
self.has_control = True
|
| 194 |
+
self.control_paths = {}
|
| 195 |
+
|
| 196 |
+
# sort image paths for matching control images properly: longer names first
|
| 197 |
+
image_paths_sorted = sorted(self.image_paths, key=lambda p: len(os.path.basename(p)), reverse=True)
|
| 198 |
+
|
| 199 |
+
# glob control images first
|
| 200 |
+
all_control_image_paths = set(glob_images(self.control_directory))
|
| 201 |
+
|
| 202 |
+
for image_path in image_paths_sorted:
|
| 203 |
+
image_basename = os.path.basename(image_path)
|
| 204 |
+
image_basename_no_ext = os.path.splitext(image_basename)[0]
|
| 205 |
+
|
| 206 |
+
# find matching control images
|
| 207 |
+
potential_paths = [
|
| 208 |
+
p
|
| 209 |
+
for p in all_control_image_paths
|
| 210 |
+
if os.path.basename(p).startswith(image_basename_no_ext + ".")
|
| 211 |
+
or os.path.basename(p).startswith(image_basename_no_ext + "_")
|
| 212 |
+
]
|
| 213 |
+
|
| 214 |
+
# remove to avoid duplicate matching
|
| 215 |
+
all_control_image_paths.difference_update(potential_paths)
|
| 216 |
+
|
| 217 |
+
if potential_paths:
|
| 218 |
+
# sort by the digits (`_0000`) suffix, prefer the one without the suffix
|
| 219 |
+
def sort_key(path):
|
| 220 |
+
basename = os.path.basename(path)
|
| 221 |
+
basename_no_ext = os.path.splitext(basename)[0]
|
| 222 |
+
if image_basename_no_ext == basename_no_ext: # prefer the one without suffix
|
| 223 |
+
return 0
|
| 224 |
+
digits_suffix = basename_no_ext.rsplit("_", 1)[-1]
|
| 225 |
+
if not digits_suffix.isdigit():
|
| 226 |
+
raise ValueError(f"Invalid digits suffix in {basename_no_ext}")
|
| 227 |
+
return int(digits_suffix) + 1
|
| 228 |
+
|
| 229 |
+
potential_paths.sort(key=sort_key)
|
| 230 |
+
if control_count_per_image is not None and len(potential_paths) < control_count_per_image:
|
| 231 |
+
logger.error(
|
| 232 |
+
f"Not enough control images for {image_path}: found {len(potential_paths)}, expected {control_count_per_image}"
|
| 233 |
+
)
|
| 234 |
+
raise ValueError(
|
| 235 |
+
f"Not enough control images for {image_path}: found {len(potential_paths)}, expected {control_count_per_image}"
|
| 236 |
+
)
|
| 237 |
+
|
| 238 |
+
# take the first `control_count_per_image` paths
|
| 239 |
+
self.control_paths[image_path] = (
|
| 240 |
+
potential_paths[:control_count_per_image] if control_count_per_image is not None else potential_paths
|
| 241 |
+
)
|
| 242 |
+
logger.info(
|
| 243 |
+
f"found {len(self.control_paths)} matching control images for {'arbitrary' if control_count_per_image is None else control_count_per_image} images"
|
| 244 |
+
)
|
| 245 |
+
|
| 246 |
+
# log the distribution of number of control images
|
| 247 |
+
count_of_num_control_images = {}
|
| 248 |
+
for paths in self.control_paths.values():
|
| 249 |
+
count = len(paths)
|
| 250 |
+
if count not in count_of_num_control_images:
|
| 251 |
+
count_of_num_control_images[count] = 0
|
| 252 |
+
count_of_num_control_images[count] += 1
|
| 253 |
+
for count, num_images in count_of_num_control_images.items():
|
| 254 |
+
logger.info(f" {num_images} images have {count} control images")
|
| 255 |
+
|
| 256 |
+
missing_controls = len(self.image_paths) - len(self.control_paths)
|
| 257 |
+
if missing_controls > 0:
|
| 258 |
+
missing_control_paths = set(self.image_paths) - set(self.control_paths.keys())
|
| 259 |
+
logger.error(f"Could not find matching control images for {missing_controls} images: {missing_control_paths}")
|
| 260 |
+
raise ValueError(f"Could not find matching control images for {missing_controls} images")
|
| 261 |
+
|
| 262 |
+
def is_indexable(self):
|
| 263 |
+
return True
|
| 264 |
+
|
| 265 |
+
def __len__(self):
|
| 266 |
+
return len(self.image_paths)
|
| 267 |
+
|
| 268 |
+
def get_image_data(self, idx: int) -> tuple[str, list[Image.Image], str, Optional[list[Image.Image]]]:
|
| 269 |
+
image_path = self.image_paths[idx]
|
| 270 |
+
image_paths = [image_path]
|
| 271 |
+
if self.multiple_target:
|
| 272 |
+
# load multiple-target images
|
| 273 |
+
image_paths += self.target_paths.get(image_path, [])
|
| 274 |
+
|
| 275 |
+
images = []
|
| 276 |
+
for p in image_paths:
|
| 277 |
+
img = Image.open(p)
|
| 278 |
+
if img.mode != "RGB" and img.mode != "RGBA":
|
| 279 |
+
img = img.convert("RGB")
|
| 280 |
+
images.append(img)
|
| 281 |
+
|
| 282 |
+
_, caption = self.get_caption(idx)
|
| 283 |
+
|
| 284 |
+
controls = None
|
| 285 |
+
if self.has_control:
|
| 286 |
+
controls = []
|
| 287 |
+
for control_path in self.control_paths[image_path]:
|
| 288 |
+
control = Image.open(control_path)
|
| 289 |
+
if control.mode != "RGB" and control.mode != "RGBA":
|
| 290 |
+
control = control.convert("RGB")
|
| 291 |
+
controls.append(control)
|
| 292 |
+
|
| 293 |
+
return image_path, images, caption, controls
|
| 294 |
+
|
| 295 |
+
def get_caption(self, idx: int) -> tuple[str, str]:
|
| 296 |
+
image_path = self.image_paths[idx]
|
| 297 |
+
caption_path = os.path.splitext(image_path)[0] + self.caption_extension if self.caption_extension else ""
|
| 298 |
+
with open(caption_path, "r", encoding="utf-8") as f:
|
| 299 |
+
caption = f.read().strip()
|
| 300 |
+
return image_path, caption
|
| 301 |
+
|
| 302 |
+
def get_control_paths(self) -> dict[str, list[str]]:
|
| 303 |
+
if not self.has_control:
|
| 304 |
+
return {}
|
| 305 |
+
return {image_path: list(paths) for image_path, paths in self.control_paths.items()}
|
| 306 |
+
|
| 307 |
+
def get_item_extras(self, idx: int) -> ItemExtras:
|
| 308 |
+
# a directory item has no place for fields beyond the shared schema
|
| 309 |
+
return ItemExtras(fields={}, base_directory=self.image_directory, label=self.image_paths[idx])
|
| 310 |
+
|
| 311 |
+
def __iter__(self):
|
| 312 |
+
self.current_idx = 0
|
| 313 |
+
return self
|
| 314 |
+
|
| 315 |
+
def __next__(self) -> callable:
|
| 316 |
+
"""
|
| 317 |
+
Returns a fetcher function that returns image data.
|
| 318 |
+
"""
|
| 319 |
+
if self.current_idx >= len(self.image_paths):
|
| 320 |
+
raise StopIteration
|
| 321 |
+
|
| 322 |
+
if self.caption_only:
|
| 323 |
+
|
| 324 |
+
def create_caption_fetcher(index):
|
| 325 |
+
return lambda: self.get_caption(index)
|
| 326 |
+
|
| 327 |
+
fetcher = create_caption_fetcher(self.current_idx)
|
| 328 |
+
else:
|
| 329 |
+
fetcher = _create_image_fetcher(self, self.current_idx)
|
| 330 |
+
|
| 331 |
+
self.current_idx += 1
|
| 332 |
+
return fetcher
|
| 333 |
+
|
| 334 |
+
|
| 335 |
+
class ImageJsonlDatasource(ImageDatasource):
|
| 336 |
+
# the shared image JSONL schema (numbered variants included); every other key is an item extra
|
| 337 |
+
SHARED_KEYS = ("image_path", "caption", "control_path")
|
| 338 |
+
|
| 339 |
+
def __init__(self, image_jsonl_file: str, control_count_per_image: Optional[int] = None, multiple_target: bool = False):
|
| 340 |
+
super().__init__()
|
| 341 |
+
self.image_jsonl_file = image_jsonl_file
|
| 342 |
+
self.control_count_per_image = control_count_per_image
|
| 343 |
+
self.multiple_target = multiple_target
|
| 344 |
+
self.current_idx = 0
|
| 345 |
+
|
| 346 |
+
# load jsonl
|
| 347 |
+
logger.info(f"load image jsonl from {self.image_jsonl_file}")
|
| 348 |
+
self.data = []
|
| 349 |
+
with open(self.image_jsonl_file, "r", encoding="utf-8") as f:
|
| 350 |
+
for line in f:
|
| 351 |
+
try:
|
| 352 |
+
data = json.loads(line)
|
| 353 |
+
except json.JSONDecodeError:
|
| 354 |
+
logger.error(f"failed to load json: {line} @ {self.image_jsonl_file}")
|
| 355 |
+
raise
|
| 356 |
+
self.data.append(data)
|
| 357 |
+
logger.info(f"loaded {len(self.data)} images")
|
| 358 |
+
|
| 359 |
+
# Normalize control paths
|
| 360 |
+
for item in self.data:
|
| 361 |
+
if "control_path" in item:
|
| 362 |
+
item["control_path_0"] = item.pop("control_path")
|
| 363 |
+
|
| 364 |
+
# Ensure control paths are named consistently, from control_path_0000 to control_path_0, control_path_1, etc.
|
| 365 |
+
control_path_keys = [key for key in item.keys() if key.startswith("control_path_")]
|
| 366 |
+
control_path_keys.sort(key=lambda x: int(x.split("_")[-1]))
|
| 367 |
+
for i, key in enumerate(control_path_keys):
|
| 368 |
+
if key != f"control_path_{i}":
|
| 369 |
+
item[f"control_path_{i}"] = item.pop(key)
|
| 370 |
+
|
| 371 |
+
# Check if there are control paths in the JSONL
|
| 372 |
+
self.has_control = any("control_path_0" in item for item in self.data)
|
| 373 |
+
if self.has_control:
|
| 374 |
+
if self.control_count_per_image is None:
|
| 375 |
+
logger.info(f"found {len(self.data)} images with arbitrary control images per image in JSONL data")
|
| 376 |
+
else:
|
| 377 |
+
missing_control_images = [
|
| 378 |
+
item["image_path"]
|
| 379 |
+
for item in self.data
|
| 380 |
+
if sum(f"control_path_{i}" not in item for i in range(self.control_count_per_image)) > 0
|
| 381 |
+
]
|
| 382 |
+
if missing_control_images:
|
| 383 |
+
logger.error(f"Some images do not have control paths in JSONL data: {missing_control_images}")
|
| 384 |
+
raise ValueError(f"Some images do not have control paths in JSONL data: {missing_control_images}")
|
| 385 |
+
logger.info(
|
| 386 |
+
f"found {len(self.data)} images with {self.control_count_per_image} control images per image in JSONL data"
|
| 387 |
+
)
|
| 388 |
+
|
| 389 |
+
def is_indexable(self):
|
| 390 |
+
return True
|
| 391 |
+
|
| 392 |
+
def __len__(self):
|
| 393 |
+
return len(self.data)
|
| 394 |
+
|
| 395 |
+
def get_image_data(self, idx: int) -> tuple[str, list[Image.Image], str, Optional[list[Image.Image]]]:
|
| 396 |
+
data = self.data[idx]
|
| 397 |
+
image_path = data.get("image_path", data.get("image_path_0"))
|
| 398 |
+
image_paths = [image_path]
|
| 399 |
+
if self.multiple_target:
|
| 400 |
+
# load multiple-target images
|
| 401 |
+
while True:
|
| 402 |
+
next_index = len(image_paths) # start from 1
|
| 403 |
+
next_image_path = data.get("image_path_" + str(next_index), None)
|
| 404 |
+
if next_image_path is None:
|
| 405 |
+
break
|
| 406 |
+
if not os.path.exists(next_image_path):
|
| 407 |
+
raise ValueError(f"multiple-target image not found: {next_image_path}")
|
| 408 |
+
|
| 409 |
+
image_paths.append(next_image_path)
|
| 410 |
+
|
| 411 |
+
images = []
|
| 412 |
+
for path in image_paths:
|
| 413 |
+
img = Image.open(path)
|
| 414 |
+
if img.mode != "RGB" and img.mode != "RGBA":
|
| 415 |
+
img = img.convert("RGB")
|
| 416 |
+
images.append(img)
|
| 417 |
+
|
| 418 |
+
caption = data["caption"]
|
| 419 |
+
|
| 420 |
+
controls = None
|
| 421 |
+
if self.has_control:
|
| 422 |
+
controls = []
|
| 423 |
+
for i in range(self.control_count_per_image or 1000): # arbitrary large number if control_count_per_image is None
|
| 424 |
+
if f"control_path_{i}" not in data:
|
| 425 |
+
break
|
| 426 |
+
control_path = data[f"control_path_{i}"]
|
| 427 |
+
control = Image.open(control_path)
|
| 428 |
+
if control.mode != "RGB" and control.mode != "RGBA":
|
| 429 |
+
control = control.convert("RGB")
|
| 430 |
+
controls.append(control)
|
| 431 |
+
|
| 432 |
+
return image_path, images, caption, controls
|
| 433 |
+
|
| 434 |
+
def get_caption(self, idx: int) -> tuple[str, str]:
|
| 435 |
+
data = self.data[idx]
|
| 436 |
+
image_path = data.get("image_path", data.get("image_path_0"))
|
| 437 |
+
caption = data["caption"]
|
| 438 |
+
return image_path, caption
|
| 439 |
+
|
| 440 |
+
def get_control_paths(self) -> dict[str, list[str]]:
|
| 441 |
+
if not self.has_control:
|
| 442 |
+
return {}
|
| 443 |
+
control_paths: dict[str, list[str]] = {}
|
| 444 |
+
for data in self.data:
|
| 445 |
+
image_path = data.get("image_path", data.get("image_path_0"))
|
| 446 |
+
paths = []
|
| 447 |
+
for i in range(self.control_count_per_image or 1000): # same bound rule as get_image_data
|
| 448 |
+
if f"control_path_{i}" not in data:
|
| 449 |
+
break
|
| 450 |
+
paths.append(data[f"control_path_{i}"])
|
| 451 |
+
control_paths[image_path] = paths
|
| 452 |
+
return control_paths
|
| 453 |
+
|
| 454 |
+
def get_item_extras(self, idx: int) -> ItemExtras:
|
| 455 |
+
return ItemExtras(
|
| 456 |
+
fields=_extra_fields(self.data[idx], ImageJsonlDatasource.SHARED_KEYS),
|
| 457 |
+
base_directory=os.path.dirname(os.path.abspath(self.image_jsonl_file)),
|
| 458 |
+
label=f"{os.path.basename(self.image_jsonl_file)} line {idx + 1}",
|
| 459 |
+
)
|
| 460 |
+
|
| 461 |
+
def __iter__(self):
|
| 462 |
+
self.current_idx = 0
|
| 463 |
+
return self
|
| 464 |
+
|
| 465 |
+
def __next__(self) -> callable:
|
| 466 |
+
if self.current_idx >= len(self.data):
|
| 467 |
+
raise StopIteration
|
| 468 |
+
|
| 469 |
+
if self.caption_only:
|
| 470 |
+
|
| 471 |
+
def create_caption_fetcher(index):
|
| 472 |
+
return lambda: self.get_caption(index)
|
| 473 |
+
|
| 474 |
+
fetcher = create_caption_fetcher(self.current_idx)
|
| 475 |
+
|
| 476 |
+
else:
|
| 477 |
+
fetcher = _create_image_fetcher(self, self.current_idx)
|
| 478 |
+
|
| 479 |
+
self.current_idx += 1
|
| 480 |
+
return fetcher
|
| 481 |
+
|
| 482 |
+
|
| 483 |
+
class VideoDatasource(ContentDatasource):
|
| 484 |
+
def __init__(self):
|
| 485 |
+
super().__init__()
|
| 486 |
+
|
| 487 |
+
# None means all frames
|
| 488 |
+
self.start_frame = None
|
| 489 |
+
self.end_frame = None
|
| 490 |
+
|
| 491 |
+
self.bucket_selector = None
|
| 492 |
+
|
| 493 |
+
self.source_fps = None
|
| 494 |
+
self.target_fps = None
|
| 495 |
+
|
| 496 |
+
# timestamp-based fps normalization (audio-capable architectures need deterministic fps)
|
| 497 |
+
self.strict_target_fps: Optional[float] = None
|
| 498 |
+
|
| 499 |
+
# audio support: set via set_audio_spec by audio-capable architectures
|
| 500 |
+
self.audio_spec: Optional[AudioSpec] = None
|
| 501 |
+
self.audio_sources: Optional[list[Optional[AudioSource]]] = None
|
| 502 |
+
|
| 503 |
+
def __len__(self):
|
| 504 |
+
raise NotImplementedError
|
| 505 |
+
|
| 506 |
+
def get_video_data_from_path(
|
| 507 |
+
self,
|
| 508 |
+
video_path: str,
|
| 509 |
+
start_frame: Optional[int] = None,
|
| 510 |
+
end_frame: Optional[int] = None,
|
| 511 |
+
bucket_selector: Optional[BucketSelector] = None,
|
| 512 |
+
) -> list[Image.Image]:
|
| 513 |
+
# this method can resize the video if bucket_selector is given to reduce the memory usage
|
| 514 |
+
|
| 515 |
+
start_frame = start_frame if start_frame is not None else self.start_frame
|
| 516 |
+
end_frame = end_frame if end_frame is not None else self.end_frame
|
| 517 |
+
bucket_selector = bucket_selector if bucket_selector is not None else self.bucket_selector
|
| 518 |
+
|
| 519 |
+
if self.strict_target_fps is not None:
|
| 520 |
+
return load_video(
|
| 521 |
+
video_path,
|
| 522 |
+
start_frame,
|
| 523 |
+
end_frame,
|
| 524 |
+
bucket_selector,
|
| 525 |
+
target_fps=self.strict_target_fps,
|
| 526 |
+
fps_resample_mode="timestamps",
|
| 527 |
+
)
|
| 528 |
+
|
| 529 |
+
video = load_video(
|
| 530 |
+
video_path, start_frame, end_frame, bucket_selector, source_fps=self.source_fps, target_fps=self.target_fps
|
| 531 |
+
)
|
| 532 |
+
return video
|
| 533 |
+
|
| 534 |
+
def get_control_data_from_path(
|
| 535 |
+
self,
|
| 536 |
+
control_path: str,
|
| 537 |
+
start_frame: Optional[int] = None,
|
| 538 |
+
end_frame: Optional[int] = None,
|
| 539 |
+
bucket_selector: Optional[BucketSelector] = None,
|
| 540 |
+
) -> list[Image.Image]:
|
| 541 |
+
start_frame = start_frame if start_frame is not None else self.start_frame
|
| 542 |
+
end_frame = end_frame if end_frame is not None else self.end_frame
|
| 543 |
+
bucket_selector = bucket_selector if bucket_selector is not None else self.bucket_selector
|
| 544 |
+
|
| 545 |
+
if self.strict_target_fps is not None:
|
| 546 |
+
return load_video(
|
| 547 |
+
control_path,
|
| 548 |
+
start_frame,
|
| 549 |
+
end_frame,
|
| 550 |
+
bucket_selector,
|
| 551 |
+
target_fps=self.strict_target_fps,
|
| 552 |
+
fps_resample_mode="timestamps",
|
| 553 |
+
)
|
| 554 |
+
|
| 555 |
+
control = load_video(
|
| 556 |
+
control_path, start_frame, end_frame, bucket_selector, source_fps=self.source_fps, target_fps=self.target_fps
|
| 557 |
+
)
|
| 558 |
+
return control
|
| 559 |
+
|
| 560 |
+
def set_start_and_end_frame(self, start_frame: Optional[int], end_frame: Optional[int]):
|
| 561 |
+
self.start_frame = start_frame
|
| 562 |
+
self.end_frame = end_frame
|
| 563 |
+
|
| 564 |
+
def set_bucket_selector(self, bucket_selector: BucketSelector):
|
| 565 |
+
self.bucket_selector = bucket_selector
|
| 566 |
+
|
| 567 |
+
def set_source_and_target_fps(self, source_fps: Optional[float], target_fps: Optional[float]):
|
| 568 |
+
self.source_fps = source_fps
|
| 569 |
+
self.target_fps = target_fps
|
| 570 |
+
|
| 571 |
+
def set_strict_target_fps(self, target_fps: Optional[float]):
|
| 572 |
+
self.strict_target_fps = target_fps
|
| 573 |
+
|
| 574 |
+
def set_audio_spec(self, audio_spec: Optional[AudioSpec]):
|
| 575 |
+
"""Enables audio for this datasource and eagerly resolves all audio sources (fail-fast)."""
|
| 576 |
+
self.audio_spec = audio_spec
|
| 577 |
+
self.audio_sources = None
|
| 578 |
+
if audio_spec is None:
|
| 579 |
+
return
|
| 580 |
+
|
| 581 |
+
audio_sources = []
|
| 582 |
+
missing = []
|
| 583 |
+
for index in range(len(self)):
|
| 584 |
+
video_path, explicit_path = self._audio_resolution_inputs(index)
|
| 585 |
+
source = resolve_audio_source(video_path, explicit_path)
|
| 586 |
+
audio_sources.append(source)
|
| 587 |
+
if source is None:
|
| 588 |
+
missing.append(video_path)
|
| 589 |
+
self.audio_sources = audio_sources
|
| 590 |
+
|
| 591 |
+
if missing:
|
| 592 |
+
for video_path in missing[:10]:
|
| 593 |
+
logger.warning(f"Video has no audio source; an unsupervised silence placeholder will be cached: {video_path}")
|
| 594 |
+
logger.info(f"audio sources resolved: {len(audio_sources) - len(missing)} with audio, {len(missing)} without")
|
| 595 |
+
|
| 596 |
+
def _audio_resolution_inputs(self, idx: int) -> tuple[str, Optional[str]]:
|
| 597 |
+
"""Returns (video_path, explicit_audio_path) for audio source resolution."""
|
| 598 |
+
raise NotImplementedError
|
| 599 |
+
|
| 600 |
+
def get_audio_waveform(self, idx: int) -> Optional[DecodedAudio]:
|
| 601 |
+
"""Decodes the item's full audio (waveform [C, L] plus timeline repairs), or None if it has no audio source."""
|
| 602 |
+
if self.audio_spec is None or self.audio_sources is None:
|
| 603 |
+
raise ValueError("Audio is not enabled for this datasource; call set_audio_spec first")
|
| 604 |
+
source = self.audio_sources[idx]
|
| 605 |
+
if source is None:
|
| 606 |
+
return None
|
| 607 |
+
return decode_audio(
|
| 608 |
+
source,
|
| 609 |
+
sample_rate=self.audio_spec.sample_rate,
|
| 610 |
+
channels=self.audio_spec.channels,
|
| 611 |
+
max_discontinuity_seconds=self.audio_spec.max_discontinuity_seconds,
|
| 612 |
+
)
|
| 613 |
+
|
| 614 |
+
def _create_video_fetcher(self, index: int):
|
| 615 |
+
if self.audio_spec is not None:
|
| 616 |
+
|
| 617 |
+
def fetch():
|
| 618 |
+
video_path, video, caption, control = self.get_video_data(index)
|
| 619 |
+
waveform = self.get_audio_waveform(index)
|
| 620 |
+
return video_path, video, caption, control, waveform
|
| 621 |
+
|
| 622 |
+
else:
|
| 623 |
+
|
| 624 |
+
def fetch():
|
| 625 |
+
return self.get_video_data(index)
|
| 626 |
+
|
| 627 |
+
# the datasource record index travels as a fetcher attribute so that ItemInfo can
|
| 628 |
+
# reference the originating record without re-deriving it from item keys
|
| 629 |
+
fetch.datasource_index = index
|
| 630 |
+
return fetch
|
| 631 |
+
|
| 632 |
+
def __iter__(self):
|
| 633 |
+
raise NotImplementedError
|
| 634 |
+
|
| 635 |
+
def __next__(self):
|
| 636 |
+
raise NotImplementedError
|
| 637 |
+
|
| 638 |
+
|
| 639 |
+
class VideoDirectoryDatasource(VideoDatasource):
|
| 640 |
+
def __init__(self, video_directory: str, caption_extension: Optional[str] = None, control_directory: Optional[str] = None):
|
| 641 |
+
super().__init__()
|
| 642 |
+
self.video_directory = video_directory
|
| 643 |
+
self.caption_extension = caption_extension
|
| 644 |
+
self.control_directory = control_directory
|
| 645 |
+
self.current_idx = 0
|
| 646 |
+
|
| 647 |
+
# glob videos
|
| 648 |
+
logger.info(f"glob videos in {self.video_directory}")
|
| 649 |
+
self.video_paths = glob_videos(self.video_directory)
|
| 650 |
+
logger.info(f"found {len(self.video_paths)} videos")
|
| 651 |
+
|
| 652 |
+
# glob control images if specified
|
| 653 |
+
if self.control_directory is not None:
|
| 654 |
+
logger.info(f"glob control videos in {self.control_directory}")
|
| 655 |
+
self.has_control = True
|
| 656 |
+
self.control_paths = {}
|
| 657 |
+
for video_path in self.video_paths:
|
| 658 |
+
video_basename = os.path.basename(video_path)
|
| 659 |
+
# construct control path from video path
|
| 660 |
+
# for example: video_path = "vid/video.mp4" -> control_path = "control/video.mp4"
|
| 661 |
+
control_path = os.path.join(self.control_directory, video_basename)
|
| 662 |
+
if os.path.exists(control_path):
|
| 663 |
+
self.control_paths[video_path] = control_path
|
| 664 |
+
else:
|
| 665 |
+
# use the same base name for control path
|
| 666 |
+
base_name = os.path.splitext(video_basename)[0]
|
| 667 |
+
|
| 668 |
+
# directory with images. for example: video_path = "vid/video.mp4" -> control_path = "control/video"
|
| 669 |
+
potential_path = os.path.join(self.control_directory, base_name) # no extension
|
| 670 |
+
if os.path.isdir(potential_path):
|
| 671 |
+
self.control_paths[video_path] = potential_path
|
| 672 |
+
else:
|
| 673 |
+
# another extension for control path
|
| 674 |
+
# for example: video_path = "vid/video.mp4" -> control_path = "control/video.mov"
|
| 675 |
+
for ext in VIDEO_EXTENSIONS:
|
| 676 |
+
potential_path = os.path.join(self.control_directory, base_name + ext)
|
| 677 |
+
if os.path.exists(potential_path):
|
| 678 |
+
self.control_paths[video_path] = potential_path
|
| 679 |
+
break
|
| 680 |
+
|
| 681 |
+
logger.info(f"found {len(self.control_paths)} matching control videos/images")
|
| 682 |
+
# check if all videos have matching control paths, if not, raise an error
|
| 683 |
+
missing_controls = len(self.video_paths) - len(self.control_paths)
|
| 684 |
+
if missing_controls > 0:
|
| 685 |
+
# logger.warning(f"Could not find matching control videos/images for {missing_controls} videos")
|
| 686 |
+
missing_controls_videos = [video_path for video_path in self.video_paths if video_path not in self.control_paths]
|
| 687 |
+
logger.error(
|
| 688 |
+
f"Could not find matching control videos/images for {missing_controls} videos: {missing_controls_videos}"
|
| 689 |
+
)
|
| 690 |
+
raise ValueError(f"Could not find matching control videos/images for {missing_controls} videos")
|
| 691 |
+
|
| 692 |
+
def is_indexable(self):
|
| 693 |
+
return True
|
| 694 |
+
|
| 695 |
+
def __len__(self):
|
| 696 |
+
return len(self.video_paths)
|
| 697 |
+
|
| 698 |
+
def get_video_data(
|
| 699 |
+
self,
|
| 700 |
+
idx: int,
|
| 701 |
+
start_frame: Optional[int] = None,
|
| 702 |
+
end_frame: Optional[int] = None,
|
| 703 |
+
bucket_selector: Optional[BucketSelector] = None,
|
| 704 |
+
) -> tuple[str, list[Image.Image], str, Optional[list[Image.Image]]]:
|
| 705 |
+
video_path = self.video_paths[idx]
|
| 706 |
+
video = self.get_video_data_from_path(video_path, start_frame, end_frame, bucket_selector)
|
| 707 |
+
|
| 708 |
+
_, caption = self.get_caption(idx)
|
| 709 |
+
|
| 710 |
+
control = None
|
| 711 |
+
if self.control_directory is not None and video_path in self.control_paths:
|
| 712 |
+
control_path = self.control_paths[video_path]
|
| 713 |
+
control = self.get_control_data_from_path(control_path, start_frame, end_frame, bucket_selector)
|
| 714 |
+
|
| 715 |
+
return video_path, video, caption, control
|
| 716 |
+
|
| 717 |
+
def get_caption(self, idx: int) -> tuple[str, str]:
|
| 718 |
+
video_path = self.video_paths[idx]
|
| 719 |
+
caption_path = os.path.splitext(video_path)[0] + self.caption_extension if self.caption_extension else ""
|
| 720 |
+
with open(caption_path, "r", encoding="utf-8") as f:
|
| 721 |
+
caption = f.read().strip()
|
| 722 |
+
return video_path, caption
|
| 723 |
+
|
| 724 |
+
def _audio_resolution_inputs(self, idx: int) -> tuple[str, Optional[str]]:
|
| 725 |
+
return self.video_paths[idx], None
|
| 726 |
+
|
| 727 |
+
def get_item_extras(self, idx: int) -> ItemExtras:
|
| 728 |
+
# a directory item has no place for fields beyond the shared schema
|
| 729 |
+
return ItemExtras(fields={}, base_directory=self.video_directory, label=self.video_paths[idx])
|
| 730 |
+
|
| 731 |
+
def __iter__(self):
|
| 732 |
+
self.current_idx = 0
|
| 733 |
+
return self
|
| 734 |
+
|
| 735 |
+
def __next__(self):
|
| 736 |
+
if self.current_idx >= len(self.video_paths):
|
| 737 |
+
raise StopIteration
|
| 738 |
+
|
| 739 |
+
if self.caption_only:
|
| 740 |
+
|
| 741 |
+
def create_caption_fetcher(index):
|
| 742 |
+
return lambda: self.get_caption(index)
|
| 743 |
+
|
| 744 |
+
fetcher = create_caption_fetcher(self.current_idx)
|
| 745 |
+
|
| 746 |
+
else:
|
| 747 |
+
fetcher = self._create_video_fetcher(self.current_idx)
|
| 748 |
+
|
| 749 |
+
self.current_idx += 1
|
| 750 |
+
return fetcher
|
| 751 |
+
|
| 752 |
+
|
| 753 |
+
class VideoJsonlDatasource(VideoDatasource):
|
| 754 |
+
PATH_KEYS = ("video_path", "control_path", "audio_path")
|
| 755 |
+
# the shared video JSONL schema; every other key is an item extra
|
| 756 |
+
SHARED_KEYS = ("video_path", "caption", "control_path", "audio_path")
|
| 757 |
+
|
| 758 |
+
def __init__(self, video_jsonl_file: str):
|
| 759 |
+
super().__init__()
|
| 760 |
+
self.video_jsonl_file = video_jsonl_file
|
| 761 |
+
self.current_idx = 0
|
| 762 |
+
|
| 763 |
+
# load jsonl
|
| 764 |
+
logger.info(f"load video jsonl from {self.video_jsonl_file}")
|
| 765 |
+
self.data = []
|
| 766 |
+
with open(self.video_jsonl_file, "r", encoding="utf-8") as f:
|
| 767 |
+
for line in f:
|
| 768 |
+
if not line.strip():
|
| 769 |
+
continue
|
| 770 |
+
data = json.loads(line)
|
| 771 |
+
self.data.append(data)
|
| 772 |
+
logger.info(f"loaded {len(self.data)} videos")
|
| 773 |
+
|
| 774 |
+
# resolve relative paths against the working directory first (the historical behavior),
|
| 775 |
+
# then against the JSONL's own directory. Whichever location matches is rewritten to an
|
| 776 |
+
# absolute path so downstream consumers (e.g. MiniMax-H3 record building) never
|
| 777 |
+
# re-resolve the value against a different base.
|
| 778 |
+
base_directory = os.path.dirname(os.path.abspath(self.video_jsonl_file))
|
| 779 |
+
for data in self.data:
|
| 780 |
+
for key in VideoJsonlDatasource.PATH_KEYS:
|
| 781 |
+
value = data.get(key)
|
| 782 |
+
if not isinstance(value, str) or not value or os.path.isabs(value):
|
| 783 |
+
continue
|
| 784 |
+
jsonl_candidate = os.path.join(base_directory, value)
|
| 785 |
+
if os.path.exists(value):
|
| 786 |
+
data[key] = os.path.abspath(value)
|
| 787 |
+
if os.path.exists(jsonl_candidate) and not os.path.samefile(value, jsonl_candidate):
|
| 788 |
+
logger.warning(
|
| 789 |
+
f"{key} {value!r} exists both relative to the working directory and to the JSONL directory; "
|
| 790 |
+
f"using the working-directory match {data[key]}"
|
| 791 |
+
)
|
| 792 |
+
elif os.path.exists(jsonl_candidate):
|
| 793 |
+
data[key] = jsonl_candidate
|
| 794 |
+
|
| 795 |
+
# Check if there are control paths in the JSONL
|
| 796 |
+
self.has_control = any("control_path" in item for item in self.data)
|
| 797 |
+
if self.has_control:
|
| 798 |
+
control_count = sum(1 for item in self.data if "control_path" in item)
|
| 799 |
+
if control_count < len(self.data):
|
| 800 |
+
missing_control_videos = [item["video_path"] for item in self.data if "control_path" not in item]
|
| 801 |
+
logger.error(f"Some videos do not have control paths in JSONL data: {missing_control_videos}")
|
| 802 |
+
raise ValueError(f"Some videos do not have control paths in JSONL data: {missing_control_videos}")
|
| 803 |
+
logger.info(f"found {control_count} control videos/images in JSONL data")
|
| 804 |
+
|
| 805 |
+
def is_indexable(self):
|
| 806 |
+
return True
|
| 807 |
+
|
| 808 |
+
def __len__(self):
|
| 809 |
+
return len(self.data)
|
| 810 |
+
|
| 811 |
+
def get_video_data(
|
| 812 |
+
self,
|
| 813 |
+
idx: int,
|
| 814 |
+
start_frame: Optional[int] = None,
|
| 815 |
+
end_frame: Optional[int] = None,
|
| 816 |
+
bucket_selector: Optional[BucketSelector] = None,
|
| 817 |
+
) -> tuple[str, list[Image.Image], str, Optional[list[Image.Image]]]:
|
| 818 |
+
data = self.data[idx]
|
| 819 |
+
video_path = data["video_path"]
|
| 820 |
+
video = self.get_video_data_from_path(video_path, start_frame, end_frame, bucket_selector)
|
| 821 |
+
|
| 822 |
+
caption = data["caption"]
|
| 823 |
+
|
| 824 |
+
control = None
|
| 825 |
+
if "control_path" in data and data["control_path"]:
|
| 826 |
+
control_path = data["control_path"]
|
| 827 |
+
control = self.get_control_data_from_path(control_path, start_frame, end_frame, bucket_selector)
|
| 828 |
+
|
| 829 |
+
return video_path, video, caption, control
|
| 830 |
+
|
| 831 |
+
def get_caption(self, idx: int) -> tuple[str, str]:
|
| 832 |
+
data = self.data[idx]
|
| 833 |
+
video_path = data["video_path"]
|
| 834 |
+
caption = data["caption"]
|
| 835 |
+
return video_path, caption
|
| 836 |
+
|
| 837 |
+
def _audio_resolution_inputs(self, idx: int) -> tuple[str, Optional[str]]:
|
| 838 |
+
data = self.data[idx]
|
| 839 |
+
return data["video_path"], data.get("audio_path")
|
| 840 |
+
|
| 841 |
+
def get_item_extras(self, idx: int) -> ItemExtras:
|
| 842 |
+
return ItemExtras(
|
| 843 |
+
fields=_extra_fields(self.data[idx], VideoJsonlDatasource.SHARED_KEYS),
|
| 844 |
+
base_directory=os.path.dirname(os.path.abspath(self.video_jsonl_file)),
|
| 845 |
+
label=f"{os.path.basename(self.video_jsonl_file)} line {idx + 1}",
|
| 846 |
+
)
|
| 847 |
+
|
| 848 |
+
def __iter__(self):
|
| 849 |
+
self.current_idx = 0
|
| 850 |
+
return self
|
| 851 |
+
|
| 852 |
+
def __next__(self):
|
| 853 |
+
if self.current_idx >= len(self.data):
|
| 854 |
+
raise StopIteration
|
| 855 |
+
|
| 856 |
+
if self.caption_only:
|
| 857 |
+
|
| 858 |
+
def create_caption_fetcher(index):
|
| 859 |
+
return lambda: self.get_caption(index)
|
| 860 |
+
|
| 861 |
+
fetcher = create_caption_fetcher(self.current_idx)
|
| 862 |
+
|
| 863 |
+
else:
|
| 864 |
+
fetcher = self._create_video_fetcher(self.current_idx)
|
| 865 |
+
|
| 866 |
+
self.current_idx += 1
|
| 867 |
+
return fetcher
|
third_party/musubi_tuner/dataset/media_utils.py
ADDED
|
@@ -0,0 +1,415 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from bisect import bisect_left
|
| 4 |
+
import glob
|
| 5 |
+
from importlib.util import find_spec
|
| 6 |
+
import math
|
| 7 |
+
import os
|
| 8 |
+
from typing import Optional, Union, TYPE_CHECKING
|
| 9 |
+
|
| 10 |
+
import numpy as np
|
| 11 |
+
from PIL import Image
|
| 12 |
+
from musubi_tuner.utils import cv2_compat # noqa: F401 - must be imported before `import cv2`
|
| 13 |
+
import cv2
|
| 14 |
+
import av
|
| 15 |
+
|
| 16 |
+
if TYPE_CHECKING:
|
| 17 |
+
from musubi_tuner.dataset.bucket import BucketSelector
|
| 18 |
+
|
| 19 |
+
import logging
|
| 20 |
+
|
| 21 |
+
logger = logging.getLogger(__name__)
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
IMAGE_EXTENSIONS = [".png", ".jpg", ".jpeg", ".webp", ".bmp", ".PNG", ".JPG", ".JPEG", ".WEBP", ".BMP", ".avif", ".AVIF"]
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
if find_spec("jxlpy") is not None: # JPEG-XL on Linux
|
| 28 |
+
from jxlpy import JXLImagePlugin # noqa: F401 # type: ignore
|
| 29 |
+
|
| 30 |
+
IMAGE_EXTENSIONS.extend([".jxl", ".JXL"])
|
| 31 |
+
|
| 32 |
+
if find_spec("pillow_jxl") is not None: # JPEG-XL on Windows
|
| 33 |
+
import pillow_jxl # noqa: F401 # type: ignore
|
| 34 |
+
|
| 35 |
+
IMAGE_EXTENSIONS.extend([".jxl", ".JXL"])
|
| 36 |
+
|
| 37 |
+
VIDEO_EXTENSIONS = [
|
| 38 |
+
".mp4",
|
| 39 |
+
".webm",
|
| 40 |
+
".avi",
|
| 41 |
+
".mkv",
|
| 42 |
+
".mov",
|
| 43 |
+
".flv",
|
| 44 |
+
".wmv",
|
| 45 |
+
".m4v",
|
| 46 |
+
".mpg",
|
| 47 |
+
".mpeg",
|
| 48 |
+
".MP4",
|
| 49 |
+
".WEBM",
|
| 50 |
+
".AVI",
|
| 51 |
+
".MKV",
|
| 52 |
+
".MOV",
|
| 53 |
+
".FLV",
|
| 54 |
+
".WMV",
|
| 55 |
+
".M4V",
|
| 56 |
+
".MPG",
|
| 57 |
+
".MPEG",
|
| 58 |
+
] # some of them are not tested
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def glob_images(directory, base="*", caption_extension=None):
|
| 62 |
+
img_paths = []
|
| 63 |
+
for ext in IMAGE_EXTENSIONS:
|
| 64 |
+
if base == "*":
|
| 65 |
+
img_paths.extend(glob.glob(os.path.join(glob.escape(directory), base + ext)))
|
| 66 |
+
else:
|
| 67 |
+
img_paths.extend(glob.glob(glob.escape(os.path.join(directory, base + ext))))
|
| 68 |
+
img_paths = list(set(img_paths)) # remove duplicates
|
| 69 |
+
|
| 70 |
+
# check for caption files and only keep images with captions
|
| 71 |
+
if caption_extension is not None:
|
| 72 |
+
caption_paths = glob.glob(os.path.join(glob.escape(directory), "*" + caption_extension))
|
| 73 |
+
caption_bases = set()
|
| 74 |
+
for caption_path in caption_paths:
|
| 75 |
+
caption_base = os.path.splitext(os.path.basename(caption_path))[0]
|
| 76 |
+
caption_bases.add(caption_base)
|
| 77 |
+
filtered_img_paths = []
|
| 78 |
+
for img_path in img_paths:
|
| 79 |
+
img_base = os.path.splitext(os.path.basename(img_path))[0]
|
| 80 |
+
if img_base in caption_bases:
|
| 81 |
+
filtered_img_paths.append(img_path)
|
| 82 |
+
img_paths = filtered_img_paths
|
| 83 |
+
|
| 84 |
+
img_paths.sort()
|
| 85 |
+
return img_paths
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
def glob_videos(directory, base="*"):
|
| 89 |
+
video_paths = []
|
| 90 |
+
for ext in VIDEO_EXTENSIONS:
|
| 91 |
+
if base == "*":
|
| 92 |
+
video_paths.extend(glob.glob(os.path.join(glob.escape(directory), base + ext)))
|
| 93 |
+
else:
|
| 94 |
+
video_paths.extend(glob.glob(glob.escape(os.path.join(directory, base + ext))))
|
| 95 |
+
video_paths = list(set(video_paths)) # remove duplicates
|
| 96 |
+
video_paths.sort()
|
| 97 |
+
return video_paths
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
def divisible_by(num: int, divisor: int) -> int:
|
| 101 |
+
return num - num % divisor
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
def resize_image_to_bucket(image: Union[Image.Image, np.ndarray], bucket_reso: tuple[int, int]) -> np.ndarray:
|
| 105 |
+
"""
|
| 106 |
+
Resize the image to the bucket resolution.
|
| 107 |
+
|
| 108 |
+
bucket_reso: **(width, height)**
|
| 109 |
+
"""
|
| 110 |
+
is_pil_image = isinstance(image, Image.Image)
|
| 111 |
+
if is_pil_image:
|
| 112 |
+
image_width, image_height = image.size
|
| 113 |
+
else:
|
| 114 |
+
image_height, image_width = image.shape[:2]
|
| 115 |
+
|
| 116 |
+
if bucket_reso == (image_width, image_height):
|
| 117 |
+
return np.array(image) if is_pil_image else image
|
| 118 |
+
|
| 119 |
+
bucket_width, bucket_height = bucket_reso
|
| 120 |
+
|
| 121 |
+
# resize the image to the bucket resolution to match the short side
|
| 122 |
+
scale_width = bucket_width / image_width
|
| 123 |
+
scale_height = bucket_height / image_height
|
| 124 |
+
scale = max(scale_width, scale_height)
|
| 125 |
+
image_width = int(image_width * scale + 0.5)
|
| 126 |
+
image_height = int(image_height * scale + 0.5)
|
| 127 |
+
|
| 128 |
+
if scale > 1:
|
| 129 |
+
image = Image.fromarray(image) if not is_pil_image else image
|
| 130 |
+
image = image.resize((image_width, image_height), Image.LANCZOS)
|
| 131 |
+
image = np.array(image)
|
| 132 |
+
else:
|
| 133 |
+
image = np.array(image) if is_pil_image else image
|
| 134 |
+
image = cv2.resize(image, (image_width, image_height), interpolation=cv2.INTER_AREA)
|
| 135 |
+
|
| 136 |
+
# crop the image to the bucket resolution
|
| 137 |
+
crop_left = (image_width - bucket_width) // 2
|
| 138 |
+
crop_top = (image_height - bucket_height) // 2
|
| 139 |
+
image = image[crop_top : crop_top + bucket_height, crop_left : crop_left + bucket_width]
|
| 140 |
+
return image
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
# A decoder may output frames up to this many positions away from their timestamp order (B-frame
|
| 144 |
+
# reordering depth plus one); resample_frame_indices reorders such frames by timestamp, while a
|
| 145 |
+
# timestamp stepping back further is a broken timeline.
|
| 146 |
+
VIDEO_REORDER_TOLERANCE_FRAMES = 4
|
| 147 |
+
|
| 148 |
+
|
| 149 |
+
def resample_frame_indices(
|
| 150 |
+
timestamps: list[float],
|
| 151 |
+
*,
|
| 152 |
+
source_frame_duration: float,
|
| 153 |
+
target_fps: float,
|
| 154 |
+
context: str = "",
|
| 155 |
+
) -> list[int]:
|
| 156 |
+
"""Maps decoded frame timestamps to nearest-frame indices on a fixed target-fps grid.
|
| 157 |
+
|
| 158 |
+
Used by fps_resample_mode="timestamps" to normalize videos of any (possibly variable)
|
| 159 |
+
frame rate to exactly target_fps, so that audio/video alignment is deterministic.
|
| 160 |
+
`timestamps` are in decoder output order; the returned indices refer to that order.
|
| 161 |
+
|
| 162 |
+
A decoder may hand out frames in an order that differs from their timestamps by a few
|
| 163 |
+
positions (a stray timestamp inside a B-frame group, or a decoder ordering by picture
|
| 164 |
+
order count while the file's timestamps disagree); presentation order is timestamp order,
|
| 165 |
+
so such frames are reordered by timestamp with a warning. A timestamp that steps back by
|
| 166 |
+
more than `VIDEO_REORDER_TOLERANCE_FRAMES` source frames is a broken timeline (a stream-copy
|
| 167 |
+
join, a timestamp wrap) and an error. `context` (the video path) only makes the messages
|
| 168 |
+
readable.
|
| 169 |
+
"""
|
| 170 |
+
if not timestamps:
|
| 171 |
+
return []
|
| 172 |
+
if source_frame_duration <= 0 or target_fps <= 0:
|
| 173 |
+
raise ValueError("Video frame durations and target FPS must be positive")
|
| 174 |
+
suffix = f": {context}" if context else ""
|
| 175 |
+
order = list(range(len(timestamps)))
|
| 176 |
+
backward_steps = [index for index in range(1, len(timestamps)) if timestamps[index] < timestamps[index - 1]]
|
| 177 |
+
if backward_steps:
|
| 178 |
+
index = max(backward_steps, key=lambda index: timestamps[index - 1] - timestamps[index])
|
| 179 |
+
left, right = timestamps[index - 1], timestamps[index]
|
| 180 |
+
if left - right > VIDEO_REORDER_TOLERANCE_FRAMES * source_frame_duration:
|
| 181 |
+
raise ValueError(
|
| 182 |
+
f"Video timestamps must be nondecreasing: frame {index} at {right:.3f}s follows frame {index - 1}"
|
| 183 |
+
f" at {left:.3f}s{suffix}. Re-mux or re-encode the video so its frame timestamps are monotonic"
|
| 184 |
+
" (e.g. ffmpeg -i in.mp4 -fps_mode cfr -c:a copy out.mp4)."
|
| 185 |
+
)
|
| 186 |
+
index = backward_steps[0]
|
| 187 |
+
logger.warning(
|
| 188 |
+
f"Video frame {index} at {timestamps[index]:.3f}s was decoded after frame {index - 1} at"
|
| 189 |
+
f" {timestamps[index - 1]:.3f}s ({len(backward_steps)} such steps); frames are reordered by timestamp{suffix}."
|
| 190 |
+
" If the result stutters, re-encode the video (e.g. ffmpeg -i in.mp4 -fps_mode cfr -c:a copy out.mp4)."
|
| 191 |
+
)
|
| 192 |
+
order.sort(key=timestamps.__getitem__) # stable: equal timestamps keep decode order
|
| 193 |
+
timestamps = [timestamps[index] for index in order]
|
| 194 |
+
|
| 195 |
+
origin = timestamps[0]
|
| 196 |
+
normalized = [timestamp - origin for timestamp in timestamps]
|
| 197 |
+
duration = normalized[-1] + source_frame_duration
|
| 198 |
+
target_count = max(1, math.ceil(duration * target_fps - 1e-9))
|
| 199 |
+
indices = []
|
| 200 |
+
for target_index in range(target_count):
|
| 201 |
+
target_time = target_index / target_fps
|
| 202 |
+
right = bisect_left(normalized, target_time)
|
| 203 |
+
if right == 0:
|
| 204 |
+
source_index = 0
|
| 205 |
+
elif right == len(normalized):
|
| 206 |
+
source_index = len(normalized) - 1
|
| 207 |
+
else:
|
| 208 |
+
left = right - 1
|
| 209 |
+
left_distance = target_time - normalized[left]
|
| 210 |
+
right_distance = normalized[right] - target_time
|
| 211 |
+
source_index = left if left_distance <= right_distance + 1e-12 else right
|
| 212 |
+
indices.append(order[source_index])
|
| 213 |
+
return indices
|
| 214 |
+
|
| 215 |
+
|
| 216 |
+
def video_origin_seconds(video_path: str) -> Optional[float]:
|
| 217 |
+
"""Timestamp of the earliest video frame: the origin of the frame grid that
|
| 218 |
+
fps_resample_mode="timestamps" builds, so anything aligned to that grid (embedded audio)
|
| 219 |
+
is placed relative to it. None if the file has no video stream or decodes no frame; a
|
| 220 |
+
frame without a timestamp sits at 0, as in the loader. Only the first few frames are
|
| 221 |
+
decoded: the loader reorders frames by timestamp within `VIDEO_REORDER_TOLERANCE_FRAMES`
|
| 222 |
+
positions, so the earliest of those is its origin."""
|
| 223 |
+
origin = None
|
| 224 |
+
with av.open(video_path) as container:
|
| 225 |
+
if not container.streams.video:
|
| 226 |
+
return None
|
| 227 |
+
for index, frame in enumerate(container.decode(container.streams.video[0])):
|
| 228 |
+
if frame.pts is None or frame.time_base is None:
|
| 229 |
+
return 0.0
|
| 230 |
+
timestamp = float(frame.pts * frame.time_base)
|
| 231 |
+
origin = timestamp if origin is None else min(origin, timestamp)
|
| 232 |
+
if index >= 2 * VIDEO_REORDER_TOLERANCE_FRAMES:
|
| 233 |
+
break
|
| 234 |
+
return origin
|
| 235 |
+
|
| 236 |
+
|
| 237 |
+
def _load_video_timestamp_resampled(
|
| 238 |
+
video_path: str,
|
| 239 |
+
target_fps: float,
|
| 240 |
+
start_frame: Optional[int],
|
| 241 |
+
end_frame: Optional[int],
|
| 242 |
+
bucket_selector: Optional[BucketSelector],
|
| 243 |
+
bucket_reso: Optional[tuple[int, int]],
|
| 244 |
+
) -> list[np.ndarray]:
|
| 245 |
+
if not os.path.isfile(video_path):
|
| 246 |
+
raise ValueError(f"fps_resample_mode='timestamps' requires a video file, not a directory: {video_path}")
|
| 247 |
+
|
| 248 |
+
with av.open(video_path) as container:
|
| 249 |
+
if not container.streams.video:
|
| 250 |
+
raise ValueError(f"Video source has no video stream: {video_path}")
|
| 251 |
+
stream = container.streams.video[0]
|
| 252 |
+
average_rate = float(stream.average_rate) if stream.average_rate is not None else target_fps
|
| 253 |
+
source_frame_duration = 1.0 / average_rate if average_rate > 0 else 1.0 / target_fps
|
| 254 |
+
frames = []
|
| 255 |
+
timestamps = []
|
| 256 |
+
for index, frame in enumerate(container.decode(stream)):
|
| 257 |
+
if frame.pts is not None and frame.time_base is not None:
|
| 258 |
+
timestamp = float(frame.pts * frame.time_base)
|
| 259 |
+
else:
|
| 260 |
+
timestamp = index * source_frame_duration
|
| 261 |
+
frames.append(frame.to_ndarray(format="rgb24"))
|
| 262 |
+
timestamps.append(timestamp)
|
| 263 |
+
if not frames:
|
| 264 |
+
raise ValueError(f"Video source decoded no frames: {video_path}")
|
| 265 |
+
|
| 266 |
+
indices = resample_frame_indices(
|
| 267 |
+
timestamps, source_frame_duration=source_frame_duration, target_fps=target_fps, context=video_path
|
| 268 |
+
)
|
| 269 |
+
indices = indices[slice(start_frame, end_frame)]
|
| 270 |
+
|
| 271 |
+
video = []
|
| 272 |
+
for index in indices:
|
| 273 |
+
frame = frames[index]
|
| 274 |
+
if bucket_selector is not None and bucket_reso is None:
|
| 275 |
+
bucket_reso = bucket_selector.get_bucket_resolution((frame.shape[1], frame.shape[0]))
|
| 276 |
+
if bucket_reso is not None:
|
| 277 |
+
frame = resize_image_to_bucket(frame, bucket_reso)
|
| 278 |
+
video.append(frame)
|
| 279 |
+
return video
|
| 280 |
+
|
| 281 |
+
|
| 282 |
+
def load_video(
|
| 283 |
+
video_path: str,
|
| 284 |
+
start_frame: Optional[int] = None,
|
| 285 |
+
end_frame: Optional[int] = None,
|
| 286 |
+
bucket_selector: Optional[BucketSelector] = None,
|
| 287 |
+
bucket_reso: Optional[tuple[int, int]] = None,
|
| 288 |
+
source_fps: Optional[float] = None,
|
| 289 |
+
target_fps: Optional[float] = None,
|
| 290 |
+
fps_resample_mode: Optional[str] = None,
|
| 291 |
+
) -> list[np.ndarray]:
|
| 292 |
+
"""
|
| 293 |
+
bucket_reso: if given, resize the video to the bucket resolution, (width, height)
|
| 294 |
+
fps_resample_mode: None (legacy source_fps/target_fps frame dropping) or "timestamps"
|
| 295 |
+
(PTS-based nearest-frame resampling to target_fps regardless of source fps)
|
| 296 |
+
"""
|
| 297 |
+
if fps_resample_mode is not None:
|
| 298 |
+
if fps_resample_mode != "timestamps":
|
| 299 |
+
raise ValueError(f"Unsupported fps_resample_mode: {fps_resample_mode}")
|
| 300 |
+
if target_fps is None:
|
| 301 |
+
raise ValueError("fps_resample_mode='timestamps' requires target_fps")
|
| 302 |
+
if source_fps is not None:
|
| 303 |
+
raise ValueError("fps_resample_mode='timestamps' does not use source_fps")
|
| 304 |
+
return _load_video_timestamp_resampled(video_path, target_fps, start_frame, end_frame, bucket_selector, bucket_reso)
|
| 305 |
+
|
| 306 |
+
if source_fps is None or target_fps is None:
|
| 307 |
+
if os.path.isfile(video_path):
|
| 308 |
+
container = av.open(video_path)
|
| 309 |
+
video = []
|
| 310 |
+
for i, frame in enumerate(container.decode(video=0)):
|
| 311 |
+
if start_frame is not None and i < start_frame:
|
| 312 |
+
continue
|
| 313 |
+
if end_frame is not None and i >= end_frame:
|
| 314 |
+
break
|
| 315 |
+
frame = frame.to_image()
|
| 316 |
+
|
| 317 |
+
if bucket_selector is not None and bucket_reso is None:
|
| 318 |
+
bucket_reso = bucket_selector.get_bucket_resolution(frame.size) # calc resolution from first frame
|
| 319 |
+
|
| 320 |
+
if bucket_reso is not None:
|
| 321 |
+
frame = resize_image_to_bucket(frame, bucket_reso)
|
| 322 |
+
else:
|
| 323 |
+
frame = np.array(frame)
|
| 324 |
+
|
| 325 |
+
video.append(frame)
|
| 326 |
+
container.close()
|
| 327 |
+
else:
|
| 328 |
+
# load images in the directory
|
| 329 |
+
image_files = glob_images(video_path)
|
| 330 |
+
image_files.sort()
|
| 331 |
+
video = []
|
| 332 |
+
for i in range(len(image_files)):
|
| 333 |
+
if start_frame is not None and i < start_frame:
|
| 334 |
+
continue
|
| 335 |
+
if end_frame is not None and i >= end_frame:
|
| 336 |
+
break
|
| 337 |
+
|
| 338 |
+
image_file = image_files[i]
|
| 339 |
+
image = Image.open(image_file).convert("RGB")
|
| 340 |
+
|
| 341 |
+
if bucket_selector is not None and bucket_reso is None:
|
| 342 |
+
bucket_reso = bucket_selector.get_bucket_resolution(image.size) # calc resolution from first frame
|
| 343 |
+
image = np.array(image)
|
| 344 |
+
if bucket_reso is not None:
|
| 345 |
+
image = resize_image_to_bucket(image, bucket_reso)
|
| 346 |
+
|
| 347 |
+
video.append(image)
|
| 348 |
+
else:
|
| 349 |
+
# drop frames to match the target fps TODO commonize this code with the above if this works
|
| 350 |
+
frame_index_delta = target_fps / source_fps # example: 16 / 30 = 0.5333
|
| 351 |
+
if os.path.isfile(video_path):
|
| 352 |
+
container = av.open(video_path)
|
| 353 |
+
video = []
|
| 354 |
+
frame_index_with_fraction = 0.0
|
| 355 |
+
previous_frame_index = -1
|
| 356 |
+
for i, frame in enumerate(container.decode(video=0)):
|
| 357 |
+
target_frame_index = int(frame_index_with_fraction)
|
| 358 |
+
frame_index_with_fraction += frame_index_delta
|
| 359 |
+
|
| 360 |
+
if target_frame_index == previous_frame_index: # drop this frame
|
| 361 |
+
continue
|
| 362 |
+
|
| 363 |
+
# accept this frame
|
| 364 |
+
previous_frame_index = target_frame_index
|
| 365 |
+
|
| 366 |
+
if start_frame is not None and target_frame_index < start_frame:
|
| 367 |
+
continue
|
| 368 |
+
if end_frame is not None and target_frame_index >= end_frame:
|
| 369 |
+
break
|
| 370 |
+
frame = frame.to_image()
|
| 371 |
+
|
| 372 |
+
if bucket_selector is not None and bucket_reso is None:
|
| 373 |
+
bucket_reso = bucket_selector.get_bucket_resolution(frame.size) # calc resolution from first frame
|
| 374 |
+
|
| 375 |
+
if bucket_reso is not None:
|
| 376 |
+
frame = resize_image_to_bucket(frame, bucket_reso)
|
| 377 |
+
else:
|
| 378 |
+
frame = np.array(frame)
|
| 379 |
+
|
| 380 |
+
video.append(frame)
|
| 381 |
+
container.close()
|
| 382 |
+
else:
|
| 383 |
+
# load images in the directory
|
| 384 |
+
image_files = glob_images(video_path)
|
| 385 |
+
image_files.sort()
|
| 386 |
+
video = []
|
| 387 |
+
frame_index_with_fraction = 0.0
|
| 388 |
+
previous_frame_index = -1
|
| 389 |
+
for i in range(len(image_files)):
|
| 390 |
+
target_frame_index = int(frame_index_with_fraction)
|
| 391 |
+
frame_index_with_fraction += frame_index_delta
|
| 392 |
+
|
| 393 |
+
if target_frame_index == previous_frame_index: # drop this frame
|
| 394 |
+
continue
|
| 395 |
+
|
| 396 |
+
# accept this frame
|
| 397 |
+
previous_frame_index = target_frame_index
|
| 398 |
+
|
| 399 |
+
if start_frame is not None and target_frame_index < start_frame:
|
| 400 |
+
continue
|
| 401 |
+
if end_frame is not None and target_frame_index >= end_frame:
|
| 402 |
+
break
|
| 403 |
+
|
| 404 |
+
image_file = image_files[i]
|
| 405 |
+
image = Image.open(image_file).convert("RGB")
|
| 406 |
+
|
| 407 |
+
if bucket_selector is not None and bucket_reso is None:
|
| 408 |
+
bucket_reso = bucket_selector.get_bucket_resolution(image.size) # calc resolution from first frame
|
| 409 |
+
image = np.array(image)
|
| 410 |
+
if bucket_reso is not None:
|
| 411 |
+
image = resize_image_to_bucket(image, bucket_reso)
|
| 412 |
+
|
| 413 |
+
video.append(image)
|
| 414 |
+
|
| 415 |
+
return video
|
third_party/musubi_tuner/minimax_h3/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
"""MiniMax-H3 model, cache, training, and sampling support."""
|
third_party/musubi_tuner/minimax_h3/checkpoint.py
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from collections.abc import Iterable
|
| 4 |
+
import hashlib
|
| 5 |
+
from pathlib import Path
|
| 6 |
+
|
| 7 |
+
import torch
|
| 8 |
+
|
| 9 |
+
from musubi_tuner.minimax_h3.media import fingerprint_file
|
| 10 |
+
from musubi_tuner.utils.safetensors_utils import MemoryEfficientSafeOpen, get_split_weight_filenames
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def resolve_safetensors_files(path: str | Path) -> list[Path]:
|
| 14 |
+
"""Resolve a checkpoint path to its safetensors files.
|
| 15 |
+
|
| 16 |
+
Repo convention: the path must be a ``.safetensors`` file; a
|
| 17 |
+
``...00001-of-00004.safetensors`` style first shard expands to all shards.
|
| 18 |
+
"""
|
| 19 |
+
path = Path(path)
|
| 20 |
+
if not path.is_file() or path.suffix != ".safetensors":
|
| 21 |
+
raise ValueError(f"MiniMax-H3 checkpoints must be a .safetensors file (pass the first shard for split checkpoints): {path}")
|
| 22 |
+
split_filenames = get_split_weight_filenames(str(path))
|
| 23 |
+
if split_filenames is not None:
|
| 24 |
+
return [Path(filename) for filename in split_filenames]
|
| 25 |
+
return [path]
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def fingerprint_checkpoint(path: str | Path) -> str:
|
| 29 |
+
"""Identity of a (possibly sharded) checkpoint for cache-staleness checks: shard names + file fingerprints."""
|
| 30 |
+
files = resolve_safetensors_files(Path(path).resolve())
|
| 31 |
+
digest = hashlib.sha256()
|
| 32 |
+
for file in files:
|
| 33 |
+
digest.update(file.name.encode("utf-8"))
|
| 34 |
+
digest.update(b"\0")
|
| 35 |
+
digest.update(fingerprint_file(file).encode("ascii"))
|
| 36 |
+
digest.update(b"\0")
|
| 37 |
+
return f"sha256:{digest.hexdigest()}"
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def load_safetensors_metadata(files: Iterable[str | Path]) -> dict[str, str]:
|
| 41 |
+
merged = {}
|
| 42 |
+
for file in files:
|
| 43 |
+
path = Path(file).resolve()
|
| 44 |
+
with MemoryEfficientSafeOpen(str(path)) as handle:
|
| 45 |
+
metadata = handle.metadata() or {}
|
| 46 |
+
for key, value in metadata.items():
|
| 47 |
+
if key in merged and merged[key] != value:
|
| 48 |
+
raise ValueError(f"Conflicting MiniMax-H3 checkpoint metadata {key!r} in {path}")
|
| 49 |
+
merged[key] = value
|
| 50 |
+
return merged
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def strip_key_prefixes(state_dict: dict[str, torch.Tensor], prefixes: tuple[str, ...]) -> dict[str, torch.Tensor]:
|
| 54 |
+
"""Strip the first matching prefix from every key (fixed provenance rules, e.g. ``vae.``)."""
|
| 55 |
+
|
| 56 |
+
def strip(key: str) -> str:
|
| 57 |
+
for prefix in prefixes:
|
| 58 |
+
if key.startswith(prefix):
|
| 59 |
+
return key[len(prefix) :]
|
| 60 |
+
return key
|
| 61 |
+
|
| 62 |
+
stripped = {strip(key): value for key, value in state_dict.items()}
|
| 63 |
+
if len(stripped) != len(state_dict):
|
| 64 |
+
raise ValueError(f"MiniMax-H3 checkpoint keys collide after stripping prefixes {prefixes}")
|
| 65 |
+
return stripped
|
third_party/musubi_tuner/minimax_h3/media.py
ADDED
|
@@ -0,0 +1,620 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from dataclasses import dataclass
|
| 4 |
+
from fractions import Fraction
|
| 5 |
+
import json
|
| 6 |
+
import math
|
| 7 |
+
from pathlib import Path
|
| 8 |
+
from typing import Callable, Literal, Mapping, Optional, Protocol, Sequence
|
| 9 |
+
|
| 10 |
+
import av
|
| 11 |
+
import numpy as np
|
| 12 |
+
from PIL import Image
|
| 13 |
+
import torch
|
| 14 |
+
|
| 15 |
+
from musubi_tuner.dataset.audio_utils import AudioSource as H3AudioSource
|
| 16 |
+
from musubi_tuner.dataset.audio_utils import AudioSpec, decode_audio, slice_audio_window
|
| 17 |
+
from musubi_tuner.dataset.datasources import ContentDatasource
|
| 18 |
+
from musubi_tuner.dataset.media_utils import load_video
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
H3Task = Literal["t2va", "fl2va", "ref2va"]
|
| 22 |
+
H3_TASKS: tuple[H3Task, ...] = ("t2va", "fl2va", "ref2va")
|
| 23 |
+
H3ReferenceType = Literal["image", "video", "audio"]
|
| 24 |
+
H3MediaProbe = Callable[[Path], "H3MediaInfo"]
|
| 25 |
+
|
| 26 |
+
TARGET_FPS = 24
|
| 27 |
+
AUDIO_SAMPLE_RATE = 32000
|
| 28 |
+
AUDIO_TERMINAL_TOLERANCE_SAMPLES = 800
|
| 29 |
+
# the released target duration range in seconds (generation and training targets; reference
|
| 30 |
+
# videos have their own 2-15 s window)
|
| 31 |
+
RELEASED_DURATION_SECONDS = (5.0, 15.0)
|
| 32 |
+
# with a one-frame (image) target, reference videos keep their full released span instead of
|
| 33 |
+
# being capped by the target duration (shared by generation and the one-frame caches)
|
| 34 |
+
ONE_FRAME_REFERENCE_FRAME_CAP = 15 * TARGET_FPS
|
| 35 |
+
# released reference canvas: short edge and pixel budget of decoded reference videos, on a
|
| 36 |
+
# 32-pixel grid (the caches and generation decode references through the same policy)
|
| 37 |
+
CANVAS_MULTIPLE = 32
|
| 38 |
+
BASE_SHORT_EDGE = 768
|
| 39 |
+
MAX_PIXELS = 768 * 1344
|
| 40 |
+
# reference videos enter the Qwen3-VL presentation as 2 fps frame samples (the released
|
| 41 |
+
# text-visual clock); the stride converts from the native 24 fps decode
|
| 42 |
+
TEXT_VISUAL_FPS = 2
|
| 43 |
+
TEXT_VISUAL_FRAME_STRIDE = TARGET_FPS // TEXT_VISUAL_FPS
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
@dataclass(frozen=True)
|
| 47 |
+
class H3MediaInfo:
|
| 48 |
+
has_audio: bool
|
| 49 |
+
duration_seconds: Optional[float]
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
@dataclass(frozen=True)
|
| 53 |
+
class H3Reference:
|
| 54 |
+
type: H3ReferenceType
|
| 55 |
+
path: Path
|
| 56 |
+
audio: Optional[H3AudioSource] = None
|
| 57 |
+
duration_seconds: Optional[float] = None
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
@dataclass(frozen=True)
|
| 61 |
+
class H3Record:
|
| 62 |
+
video_path: Path
|
| 63 |
+
caption: str
|
| 64 |
+
references: tuple[H3Reference, ...]
|
| 65 |
+
# where the record came from (e.g. "items.jsonl line 3"), for error messages only
|
| 66 |
+
label: str = ""
|
| 67 |
+
# optional per-item `teacher_caption`: the caption of the subject-reference teacher presentation
|
| 68 |
+
# (text cache only); None means the teacher wraps `caption` with the boilerplate declaration
|
| 69 |
+
teacher_caption: Optional[str] = None
|
| 70 |
+
|
| 71 |
+
@property
|
| 72 |
+
def context(self) -> str:
|
| 73 |
+
return f"H3 {self.label}" if self.label else "H3 record"
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def _parse_teacher_caption(fields: Mapping[str, object], context: str) -> Optional[str]:
|
| 77 |
+
teacher_caption = fields.get("teacher_caption")
|
| 78 |
+
if teacher_caption is not None and (not isinstance(teacher_caption, str) or not teacher_caption.strip()):
|
| 79 |
+
raise ValueError(f"{context}: teacher_caption must be a non-empty string when present")
|
| 80 |
+
return teacher_caption
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
def validate_subject_reference_record(record: H3Record, context: str) -> None:
|
| 84 |
+
"""The subject-reference teacher (v1) takes 1..9 image references; video/audio references
|
| 85 |
+
need per-item role declarations (motion? voice?) and are deferred."""
|
| 86 |
+
if not record.references:
|
| 87 |
+
raise ValueError(f"{context}: the subject-reference teacher requires at least one image reference")
|
| 88 |
+
unsupported = [reference.type for reference in record.references if reference.type != "image"]
|
| 89 |
+
if unsupported:
|
| 90 |
+
raise ValueError(
|
| 91 |
+
f"{context}: the subject-reference teacher supports image references only (got {unsupported[0]!r};"
|
| 92 |
+
" video/audio references are not supported yet)"
|
| 93 |
+
)
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
def _validate_frame_count(frame_count: int) -> None:
|
| 97 |
+
if frame_count < 5 or (frame_count - 5) % 17 != 0:
|
| 98 |
+
raise ValueError(f"Invalid MiniMax-H3 frame count {frame_count}; expected 17*n+5")
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
def video_latent_frames(frame_count: int) -> int:
|
| 102 |
+
_validate_frame_count(frame_count)
|
| 103 |
+
return 5 * ((frame_count - 5) // 17) + 2
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
def audio_latent_frames(frame_count: int, *, output_fps: int = TARGET_FPS) -> int:
|
| 107 |
+
_validate_frame_count(frame_count)
|
| 108 |
+
if isinstance(output_fps, bool) or not isinstance(output_fps, int) or output_fps <= 0:
|
| 109 |
+
raise ValueError(f"MiniMax-H3 output fps must be a positive integer, got {output_fps!r}")
|
| 110 |
+
if output_fps == TARGET_FPS:
|
| 111 |
+
return (10 * frame_count + 3) // 6
|
| 112 |
+
# 40 Hz audio latents over the real duration frame_count/output_fps seconds;
|
| 113 |
+
# reduces to the released (10*f+3)//6 mapping at 24 fps
|
| 114 |
+
return int((Fraction(10 * frame_count * TARGET_FPS, output_fps) + 3) // 6)
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
def waveform_samples(audio_frames: int) -> int:
|
| 118 |
+
if audio_frames <= 0:
|
| 119 |
+
raise ValueError(f"Audio latent frame count must be positive, got {audio_frames}")
|
| 120 |
+
return audio_frames * 800
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
def h3_samples_per_crop(frame_count: int) -> int:
|
| 124 |
+
# module-level (not a lambda) so the spec stays picklable for spawned DataLoader workers
|
| 125 |
+
return waveform_samples(audio_latent_frames(frame_count))
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
# passed to the shared dataset layer so that it decodes and windows target audio for us
|
| 129 |
+
H3_AUDIO_SPEC = AudioSpec(
|
| 130 |
+
sample_rate=AUDIO_SAMPLE_RATE,
|
| 131 |
+
channels=2,
|
| 132 |
+
samples_per_crop=h3_samples_per_crop,
|
| 133 |
+
codec_pad_tolerance=AUDIO_TERMINAL_TOLERANCE_SAMPLES,
|
| 134 |
+
)
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
def probe_h3_media(path: Path) -> H3MediaInfo:
|
| 138 |
+
with av.open(str(path)) as container:
|
| 139 |
+
has_audio = bool(container.streams.audio)
|
| 140 |
+
durations = []
|
| 141 |
+
for stream in (*container.streams.video, *container.streams.audio):
|
| 142 |
+
if stream.duration is not None and stream.time_base is not None:
|
| 143 |
+
durations.append(float(stream.duration * stream.time_base))
|
| 144 |
+
if durations:
|
| 145 |
+
duration_seconds = max(durations)
|
| 146 |
+
elif container.duration is not None:
|
| 147 |
+
duration_seconds = float(container.duration / av.time_base)
|
| 148 |
+
else:
|
| 149 |
+
duration_seconds = None
|
| 150 |
+
return H3MediaInfo(has_audio=has_audio, duration_seconds=duration_seconds)
|
| 151 |
+
|
| 152 |
+
|
| 153 |
+
def _resolve_existing_path(value: object, base_directory: Path, field: str, context: str) -> Path:
|
| 154 |
+
if not isinstance(value, str) or not value.strip():
|
| 155 |
+
raise ValueError(f"{context}: {field} must be a non-empty path")
|
| 156 |
+
path = Path(value)
|
| 157 |
+
if not path.is_absolute():
|
| 158 |
+
path = base_directory / path
|
| 159 |
+
path = path.resolve()
|
| 160 |
+
if not path.is_file():
|
| 161 |
+
raise ValueError(f"{context}: {field} does not exist: {path}")
|
| 162 |
+
return path
|
| 163 |
+
|
| 164 |
+
|
| 165 |
+
def _probe_required_audio(path: Path, probe: H3MediaProbe, label: str, context: str) -> H3MediaInfo:
|
| 166 |
+
try:
|
| 167 |
+
info = probe(path)
|
| 168 |
+
except Exception as error:
|
| 169 |
+
raise ValueError(f"{context}: {label} failed to decode: {error}") from error
|
| 170 |
+
if not info.has_audio:
|
| 171 |
+
raise ValueError(f"{context}: {label} contains no audio stream: {path}")
|
| 172 |
+
return info
|
| 173 |
+
|
| 174 |
+
|
| 175 |
+
def _validate_reference_counts(references: list, context: str) -> None:
|
| 176 |
+
if len(references) > 12:
|
| 177 |
+
raise ValueError(f"{context}: Ref2VA allows at most 12 reference items")
|
| 178 |
+
|
| 179 |
+
types = [reference.get("type") if isinstance(reference, dict) else None for reference in references]
|
| 180 |
+
unsupported = [reference_type for reference_type in types if reference_type not in {"image", "video", "audio"}]
|
| 181 |
+
if unsupported:
|
| 182 |
+
raise ValueError(f"{context}: Unsupported reference type: {unsupported[0]!r}")
|
| 183 |
+
if types.count("image") > 9:
|
| 184 |
+
raise ValueError(f"{context}: Ref2VA allows at most 9 image references")
|
| 185 |
+
if types.count("video") > 3:
|
| 186 |
+
raise ValueError(f"{context}: Ref2VA allows at most 3 video references")
|
| 187 |
+
if types.count("audio") > 3:
|
| 188 |
+
raise ValueError(f"{context}: Ref2VA allows at most 3 audio-bearing references")
|
| 189 |
+
if not any(reference_type in {"image", "video"} for reference_type in types):
|
| 190 |
+
raise ValueError(f"{context}: Ref2VA requires at least one visual reference")
|
| 191 |
+
|
| 192 |
+
|
| 193 |
+
def _parse_references(
|
| 194 |
+
raw_references: object,
|
| 195 |
+
base_directory: Path,
|
| 196 |
+
context: str,
|
| 197 |
+
probe: H3MediaProbe,
|
| 198 |
+
) -> tuple[H3Reference, ...]:
|
| 199 |
+
if not isinstance(raw_references, list):
|
| 200 |
+
raise ValueError(f"{context}: references must be a list")
|
| 201 |
+
_validate_reference_counts(raw_references, context)
|
| 202 |
+
|
| 203 |
+
references = []
|
| 204 |
+
audio_bearing_count = 0
|
| 205 |
+
for index, raw_reference in enumerate(raw_references):
|
| 206 |
+
reference_type = raw_reference["type"]
|
| 207 |
+
field_prefix = f"references[{index}]"
|
| 208 |
+
path = _resolve_existing_path(raw_reference.get("path"), base_directory, f"{field_prefix}.path", context)
|
| 209 |
+
|
| 210 |
+
if reference_type == "image":
|
| 211 |
+
if "audio_path" in raw_reference:
|
| 212 |
+
raise ValueError(f"{context}: {field_prefix} image cannot have audio_path")
|
| 213 |
+
references.append(H3Reference(type="image", path=path))
|
| 214 |
+
continue
|
| 215 |
+
|
| 216 |
+
if reference_type == "audio":
|
| 217 |
+
if "audio_path" in raw_reference:
|
| 218 |
+
raise ValueError(f"{context}: {field_prefix} audio uses path, not audio_path")
|
| 219 |
+
info = _probe_required_audio(path, probe, f"{field_prefix} audio", context)
|
| 220 |
+
references.append(
|
| 221 |
+
H3Reference(
|
| 222 |
+
type="audio",
|
| 223 |
+
path=path,
|
| 224 |
+
audio=H3AudioSource(path=path, embedded=False),
|
| 225 |
+
duration_seconds=info.duration_seconds,
|
| 226 |
+
)
|
| 227 |
+
)
|
| 228 |
+
audio_bearing_count += 1
|
| 229 |
+
continue
|
| 230 |
+
|
| 231 |
+
try:
|
| 232 |
+
video_info = probe(path)
|
| 233 |
+
except Exception as error:
|
| 234 |
+
raise ValueError(f"{context}: {field_prefix} video failed to decode: {error}") from error
|
| 235 |
+
duration = video_info.duration_seconds
|
| 236 |
+
if duration is None or duration < 2.0 or duration > 15.0:
|
| 237 |
+
raise ValueError(f"{context}: {field_prefix} video must be between 2 and 15 seconds; got {duration}")
|
| 238 |
+
|
| 239 |
+
# an explicit "audio_path": null makes the reference visual-only (e.g. a motion
|
| 240 |
+
# reference), suppressing the video's embedded audio track
|
| 241 |
+
audio = None
|
| 242 |
+
if "audio_path" not in raw_reference:
|
| 243 |
+
if video_info.has_audio:
|
| 244 |
+
audio = H3AudioSource(path=path, embedded=True)
|
| 245 |
+
elif raw_reference["audio_path"] is not None:
|
| 246 |
+
audio_path = _resolve_existing_path(raw_reference["audio_path"], base_directory, f"{field_prefix}.audio_path", context)
|
| 247 |
+
_probe_required_audio(audio_path, probe, f"Explicit {field_prefix} audio", context)
|
| 248 |
+
audio = H3AudioSource(path=audio_path, embedded=False)
|
| 249 |
+
if audio is not None:
|
| 250 |
+
audio_bearing_count += 1
|
| 251 |
+
references.append(H3Reference(type="video", path=path, audio=audio, duration_seconds=duration))
|
| 252 |
+
|
| 253 |
+
if audio_bearing_count > 3:
|
| 254 |
+
raise ValueError(f"{context}: Ref2VA allows at most 3 audio-bearing references")
|
| 255 |
+
return tuple(references)
|
| 256 |
+
|
| 257 |
+
|
| 258 |
+
INLINE_REFERENCE_IMAGE_SUFFIXES = frozenset({".bmp", ".jpeg", ".jpg", ".png", ".webp"})
|
| 259 |
+
INLINE_REFERENCE_AUDIO_SUFFIXES = frozenset({".aac", ".flac", ".m4a", ".mp3", ".ogg", ".opus", ".wav"})
|
| 260 |
+
|
| 261 |
+
|
| 262 |
+
def _inline_reference_data(spec: str, context: str) -> dict:
|
| 263 |
+
"""Parses one inline reference spec `path[;type=...][;audio=...]` into the JSONL dict shape."""
|
| 264 |
+
parts = spec.split(";")
|
| 265 |
+
path = parts[0].strip()
|
| 266 |
+
if not path:
|
| 267 |
+
raise ValueError(f"{context}: inline reference must start with a path: {spec!r}")
|
| 268 |
+
|
| 269 |
+
options: dict[str, str] = {}
|
| 270 |
+
for part in parts[1:]:
|
| 271 |
+
key, separator, value = part.partition("=")
|
| 272 |
+
key = key.strip()
|
| 273 |
+
value = value.strip()
|
| 274 |
+
if not separator or not key or not value:
|
| 275 |
+
raise ValueError(f"{context}: inline reference options must be key=value, got {part!r} in {spec!r}")
|
| 276 |
+
if key not in {"type", "audio"}:
|
| 277 |
+
raise ValueError(f"{context}: unknown inline reference option {key!r} in {spec!r} (allowed: type, audio)")
|
| 278 |
+
if key in options:
|
| 279 |
+
raise ValueError(f"{context}: duplicate inline reference option {key!r} in {spec!r}")
|
| 280 |
+
options[key] = value
|
| 281 |
+
|
| 282 |
+
reference_type = options.get("type")
|
| 283 |
+
if reference_type is None:
|
| 284 |
+
suffix = Path(path).suffix.lower()
|
| 285 |
+
if suffix in INLINE_REFERENCE_IMAGE_SUFFIXES:
|
| 286 |
+
reference_type = "image"
|
| 287 |
+
elif suffix in INLINE_REFERENCE_AUDIO_SUFFIXES:
|
| 288 |
+
reference_type = "audio"
|
| 289 |
+
else:
|
| 290 |
+
reference_type = "video"
|
| 291 |
+
elif reference_type not in {"image", "video", "audio"}:
|
| 292 |
+
raise ValueError(f"{context}: inline reference type must be image, video, or audio, got {reference_type!r}")
|
| 293 |
+
|
| 294 |
+
data: dict = {"type": reference_type, "path": path}
|
| 295 |
+
if "audio" in options:
|
| 296 |
+
# the JSONL-shape validation rejects audio_path on image and audio references
|
| 297 |
+
data["audio_path"] = options["audio"]
|
| 298 |
+
return data
|
| 299 |
+
|
| 300 |
+
|
| 301 |
+
def parse_inline_references(
|
| 302 |
+
specs: list[str] | tuple[str, ...],
|
| 303 |
+
base_directory: Path,
|
| 304 |
+
probe: H3MediaProbe = probe_h3_media,
|
| 305 |
+
context: str = "H3 --ref",
|
| 306 |
+
) -> tuple[H3Reference, ...]:
|
| 307 |
+
"""Parses inline `--ref` specs with exactly the JSONL `references` validation rules.
|
| 308 |
+
|
| 309 |
+
Relative paths resolve from base_directory (the caller decides: CWD for the CLI,
|
| 310 |
+
the prompt file's directory for sample-prompt lines).
|
| 311 |
+
"""
|
| 312 |
+
raw_references = [_inline_reference_data(spec, f"{context}[{index}]") for index, spec in enumerate(specs)]
|
| 313 |
+
return _parse_references(raw_references, base_directory, context, probe)
|
| 314 |
+
|
| 315 |
+
|
| 316 |
+
def round_to_canvas_multiple(value: float, multiple: int = CANVAS_MULTIPLE) -> int:
|
| 317 |
+
return max(multiple, round(value / multiple) * multiple)
|
| 318 |
+
|
| 319 |
+
|
| 320 |
+
def adapt_reference_canvas(width: int, height: int) -> tuple[int, int]:
|
| 321 |
+
"""The released reference-video canvas for a source aspect ratio: BASE_SHORT_EDGE on the
|
| 322 |
+
short side, scaled down to MAX_PIXELS, on the CANVAS_MULTIPLE grid."""
|
| 323 |
+
ratio = width / height
|
| 324 |
+
if ratio >= 1.0:
|
| 325 |
+
nominal_width, nominal_height = BASE_SHORT_EDGE * ratio, BASE_SHORT_EDGE
|
| 326 |
+
else:
|
| 327 |
+
nominal_width, nominal_height = BASE_SHORT_EDGE, BASE_SHORT_EDGE / ratio
|
| 328 |
+
if nominal_width * nominal_height > MAX_PIXELS:
|
| 329 |
+
scale = math.sqrt(MAX_PIXELS / (nominal_width * nominal_height))
|
| 330 |
+
nominal_width *= scale
|
| 331 |
+
nominal_height *= scale
|
| 332 |
+
return round_to_canvas_multiple(nominal_width), round_to_canvas_multiple(nominal_height)
|
| 333 |
+
|
| 334 |
+
|
| 335 |
+
def resize_frames(frames: Sequence[np.ndarray], size: tuple[int, int]) -> torch.Tensor:
|
| 336 |
+
"""LANCZOS-resize decoded RGB(A) frames to exactly (width, height) as a uint8 [F,H,W,3] tensor."""
|
| 337 |
+
width, height = size
|
| 338 |
+
resized = [
|
| 339 |
+
torch.from_numpy(np.asarray(Image.fromarray(frame[..., :3]).resize((width, height), Image.Resampling.LANCZOS)).copy())
|
| 340 |
+
for frame in frames
|
| 341 |
+
]
|
| 342 |
+
return torch.stack(resized)
|
| 343 |
+
|
| 344 |
+
|
| 345 |
+
class H3MediaDecoder(Protocol):
|
| 346 |
+
def decode_audio(
|
| 347 |
+
self,
|
| 348 |
+
source: H3AudioSource,
|
| 349 |
+
*,
|
| 350 |
+
start_sample: int,
|
| 351 |
+
sample_count: int,
|
| 352 |
+
require_exact: bool,
|
| 353 |
+
) -> torch.Tensor: ...
|
| 354 |
+
|
| 355 |
+
def decode_reference_visual(
|
| 356 |
+
self,
|
| 357 |
+
reference: H3Reference,
|
| 358 |
+
*,
|
| 359 |
+
target_frame_count: int,
|
| 360 |
+
target_size: tuple[int, int],
|
| 361 |
+
) -> torch.Tensor: ...
|
| 362 |
+
|
| 363 |
+
|
| 364 |
+
class PyAVH3MediaDecoder:
|
| 365 |
+
"""Decodes MiniMax-H3 reference media (target media is decoded by the shared dataset layer)."""
|
| 366 |
+
|
| 367 |
+
def __init__(self, audio_spec: AudioSpec = H3_AUDIO_SPEC):
|
| 368 |
+
# the same spec the shared dataset layer decodes target audio with, so reference audio
|
| 369 |
+
# gets the same tolerances (cache scripts pass the spec with the command-line overrides)
|
| 370 |
+
self.audio_spec = audio_spec
|
| 371 |
+
|
| 372 |
+
def decode_audio(
|
| 373 |
+
self,
|
| 374 |
+
source: H3AudioSource,
|
| 375 |
+
*,
|
| 376 |
+
start_sample: int,
|
| 377 |
+
sample_count: int,
|
| 378 |
+
require_exact: bool,
|
| 379 |
+
) -> torch.Tensor:
|
| 380 |
+
if start_sample < 0 or sample_count <= 0:
|
| 381 |
+
raise ValueError("MiniMax-H3 audio window must have a nonnegative start and positive length")
|
| 382 |
+
spec = self.audio_spec
|
| 383 |
+
decoded = decode_audio(
|
| 384 |
+
source,
|
| 385 |
+
sample_rate=spec.sample_rate,
|
| 386 |
+
channels=spec.channels,
|
| 387 |
+
max_discontinuity_seconds=spec.max_discontinuity_seconds,
|
| 388 |
+
)
|
| 389 |
+
return slice_audio_window(
|
| 390 |
+
decoded.waveform,
|
| 391 |
+
start_sample=start_sample,
|
| 392 |
+
sample_count=sample_count,
|
| 393 |
+
pad_tolerance=spec.codec_pad_tolerance,
|
| 394 |
+
require_exact=require_exact,
|
| 395 |
+
context=str(source.path),
|
| 396 |
+
repairs=decoded.repairs,
|
| 397 |
+
max_repair_samples=spec.max_missing_samples,
|
| 398 |
+
)
|
| 399 |
+
|
| 400 |
+
def decode_reference_visual(
|
| 401 |
+
self,
|
| 402 |
+
reference: H3Reference,
|
| 403 |
+
*,
|
| 404 |
+
target_frame_count: int,
|
| 405 |
+
target_size: tuple[int, int],
|
| 406 |
+
) -> torch.Tensor:
|
| 407 |
+
if reference.type == "image":
|
| 408 |
+
with Image.open(reference.path) as image:
|
| 409 |
+
frame = np.asarray(image.convert("RGB"))
|
| 410 |
+
height, width = frame.shape[:2]
|
| 411 |
+
target_area = target_size[0] * target_size[1]
|
| 412 |
+
scale = min(1.0, math.sqrt(target_area / (width * height)))
|
| 413 |
+
size = round_to_canvas_multiple(width * scale), round_to_canvas_multiple(height * scale)
|
| 414 |
+
return resize_frames([frame], size)
|
| 415 |
+
|
| 416 |
+
if reference.type != "video":
|
| 417 |
+
raise ValueError(f"Reference type {reference.type!r} has no visual stream")
|
| 418 |
+
frames = load_video(str(reference.path), target_fps=TARGET_FPS, fps_resample_mode="timestamps")
|
| 419 |
+
usable_frames = min(len(frames), target_frame_count)
|
| 420 |
+
if usable_frames < 5:
|
| 421 |
+
raise ValueError(f"MiniMax-H3 reference video requires at least 5 frames: {reference.path}")
|
| 422 |
+
usable_frames = 5 + ((usable_frames - 5) // 17) * 17
|
| 423 |
+
frames = frames[:usable_frames]
|
| 424 |
+
source_height, source_width = frames[0].shape[:2]
|
| 425 |
+
width, height = adapt_reference_canvas(source_width, source_height)
|
| 426 |
+
if source_width * source_height < width * height:
|
| 427 |
+
width = round_to_canvas_multiple(source_width)
|
| 428 |
+
height = round_to_canvas_multiple(source_height)
|
| 429 |
+
return resize_frames(frames, (width, height))
|
| 430 |
+
|
| 431 |
+
|
| 432 |
+
def prepare_pixels(frames: torch.Tensor | np.ndarray) -> torch.Tensor:
|
| 433 |
+
"""Decoded [F,H,W,C] pixels (uint8, or floats in [0,1]) to the VAE's [1,3,F,H,W] in [-1,1]; alpha is dropped."""
|
| 434 |
+
frames = torch.as_tensor(frames)
|
| 435 |
+
if frames.ndim != 4 or frames.shape[-1] < 3:
|
| 436 |
+
raise ValueError(f"MiniMax-H3 decoded video must be [F,H,W,C], got {tuple(frames.shape)}")
|
| 437 |
+
frames = frames[..., :3]
|
| 438 |
+
if frames.dtype == torch.uint8:
|
| 439 |
+
frames = frames.float().div_(127.5).sub_(1.0)
|
| 440 |
+
elif frames.is_floating_point():
|
| 441 |
+
if not torch.all((frames >= 0) & (frames <= 1)):
|
| 442 |
+
raise ValueError("Floating MiniMax-H3 decoded pixels must be in [0,1]")
|
| 443 |
+
frames = frames.float().mul_(2.0).sub_(1.0)
|
| 444 |
+
else:
|
| 445 |
+
raise ValueError(f"Unsupported MiniMax-H3 decoded pixel dtype: {frames.dtype}")
|
| 446 |
+
return frames.permute(3, 0, 1, 2).unsqueeze(0).contiguous()
|
| 447 |
+
|
| 448 |
+
|
| 449 |
+
def module_device_dtype(module: torch.nn.Module, fallback_dtype: torch.dtype) -> tuple[torch.device, torch.dtype]:
|
| 450 |
+
"""Device and floating dtype of a module's first floating tensor (the VAEs move between devices)."""
|
| 451 |
+
for tensor in (*module.parameters(), *module.buffers()):
|
| 452 |
+
if tensor.is_floating_point():
|
| 453 |
+
return tensor.device, tensor.dtype
|
| 454 |
+
return torch.device("cpu"), fallback_dtype
|
| 455 |
+
|
| 456 |
+
|
| 457 |
+
def fingerprint_file(path: str | Path) -> str:
|
| 458 |
+
"""Lightweight file identity (size + mtime) for cache-staleness checks; deliberately not a content hash."""
|
| 459 |
+
stat = Path(path).resolve().stat()
|
| 460 |
+
return f"stat:{stat.st_size}:{stat.st_mtime_ns}"
|
| 461 |
+
|
| 462 |
+
|
| 463 |
+
def reject_one_frame_audio_references(record: H3Record) -> None:
|
| 464 |
+
"""Standalone audio references have no window with a one-frame target (it is defined by the
|
| 465 |
+
target duration); video references keep their embedded audio. Shared by generation, the
|
| 466 |
+
one-frame caches, and training-time samples."""
|
| 467 |
+
if any(reference.type == "audio" for reference in record.references):
|
| 468 |
+
raise ValueError(
|
| 469 |
+
"MiniMax-H3 one-frame targets do not accept standalone audio references"
|
| 470 |
+
" (their window is defined by the target duration); video references keep their embedded audio"
|
| 471 |
+
)
|
| 472 |
+
|
| 473 |
+
|
| 474 |
+
def _record_from_fields(
|
| 475 |
+
*,
|
| 476 |
+
target_path: Path,
|
| 477 |
+
caption: object,
|
| 478 |
+
fields: Mapping[str, object],
|
| 479 |
+
base_directory: Path,
|
| 480 |
+
label: str,
|
| 481 |
+
task: H3Task,
|
| 482 |
+
probe: H3MediaProbe,
|
| 483 |
+
control_images: Sequence[str] | None = None,
|
| 484 |
+
) -> H3Record:
|
| 485 |
+
"""Builds a record from a validated target path plus the item's H3-specific fields
|
| 486 |
+
(``references``, ``teacher_caption``); relative reference paths resolve from base_directory.
|
| 487 |
+
|
| 488 |
+
With ``control_images`` (the item's control image paths, as the dataset layer opens them,
|
| 489 |
+
i.e. relative to the working directory) and task ref2va, a record without a ``references``
|
| 490 |
+
field takes those images as its ordered image references; a record cannot have both.
|
| 491 |
+
"""
|
| 492 |
+
context = f"H3 {label}"
|
| 493 |
+
if not isinstance(caption, str):
|
| 494 |
+
raise ValueError(f"{context}: caption must be a string")
|
| 495 |
+
|
| 496 |
+
raw_references = fields.get("references", [])
|
| 497 |
+
if task == "ref2va":
|
| 498 |
+
if control_images and "references" in fields:
|
| 499 |
+
raise ValueError(f"{context}: cannot combine control images with references")
|
| 500 |
+
if control_images:
|
| 501 |
+
raw_references = [{"type": "image", "path": str(Path(path).expanduser().resolve())} for path in control_images]
|
| 502 |
+
references = _parse_references(raw_references, base_directory, context, probe)
|
| 503 |
+
else:
|
| 504 |
+
if raw_references:
|
| 505 |
+
raise ValueError(f"{context}: references require task ref2va")
|
| 506 |
+
references = ()
|
| 507 |
+
|
| 508 |
+
return H3Record(
|
| 509 |
+
video_path=target_path,
|
| 510 |
+
caption=caption,
|
| 511 |
+
references=references,
|
| 512 |
+
label=label,
|
| 513 |
+
teacher_caption=_parse_teacher_caption(fields, context),
|
| 514 |
+
)
|
| 515 |
+
|
| 516 |
+
|
| 517 |
+
def load_h3_jsonl_records(
|
| 518 |
+
jsonl_path: str | Path,
|
| 519 |
+
task: H3Task,
|
| 520 |
+
probe: H3MediaProbe = probe_h3_media,
|
| 521 |
+
) -> list[H3Record]:
|
| 522 |
+
"""Reads a standalone Ref2VA JSONL (generation --reference_jsonl); relative paths resolve from its directory."""
|
| 523 |
+
if task not in {"t2va", "fl2va", "ref2va"}:
|
| 524 |
+
raise ValueError(f"Unsupported MiniMax-H3 task: {task}")
|
| 525 |
+
|
| 526 |
+
jsonl_path = Path(jsonl_path).resolve()
|
| 527 |
+
if not jsonl_path.is_file():
|
| 528 |
+
raise ValueError(f"MiniMax-H3 JSONL does not exist: {jsonl_path}")
|
| 529 |
+
base_directory = jsonl_path.parent
|
| 530 |
+
records = []
|
| 531 |
+
|
| 532 |
+
with jsonl_path.open("r", encoding="utf-8") as handle:
|
| 533 |
+
for line_number, line in enumerate(handle, start=1):
|
| 534 |
+
if not line.strip():
|
| 535 |
+
continue
|
| 536 |
+
label = f"{jsonl_path.name} line {line_number}"
|
| 537 |
+
context = f"H3 {label}"
|
| 538 |
+
try:
|
| 539 |
+
data = json.loads(line)
|
| 540 |
+
except json.JSONDecodeError as error:
|
| 541 |
+
raise ValueError(f"{context}: invalid JSON: {error.msg}") from error
|
| 542 |
+
if not isinstance(data, dict):
|
| 543 |
+
raise ValueError(f"{context}: each record must be an object")
|
| 544 |
+
records.append(
|
| 545 |
+
_record_from_fields(
|
| 546 |
+
target_path=_resolve_existing_path(data.get("video_path"), base_directory, "video_path", context),
|
| 547 |
+
caption=data.get("caption"),
|
| 548 |
+
fields=data,
|
| 549 |
+
base_directory=base_directory,
|
| 550 |
+
label=label,
|
| 551 |
+
task=task,
|
| 552 |
+
probe=probe,
|
| 553 |
+
)
|
| 554 |
+
)
|
| 555 |
+
|
| 556 |
+
if not records:
|
| 557 |
+
raise ValueError(f"MiniMax-H3 JSONL contains no records: {jsonl_path}")
|
| 558 |
+
return records
|
| 559 |
+
|
| 560 |
+
|
| 561 |
+
def h3_records_from_datasource(
|
| 562 |
+
datasource: ContentDatasource,
|
| 563 |
+
task: H3Task,
|
| 564 |
+
probe: H3MediaProbe = probe_h3_media,
|
| 565 |
+
*,
|
| 566 |
+
control_images_as_references: bool = False,
|
| 567 |
+
) -> list[H3Record]:
|
| 568 |
+
"""Builds the H3 records of a dataset's datasource (image or video), aligned with the
|
| 569 |
+
datasource indices so cache items find theirs through ItemInfo.datasource_index.
|
| 570 |
+
|
| 571 |
+
The target path and caption come from the shared accessor; the H3-specific fields
|
| 572 |
+
(``references``, ``teacher_caption``) come from the item extras, which only record-based
|
| 573 |
+
datasources (``video_jsonl_file`` / ``image_jsonl_file``) can carry. With
|
| 574 |
+
``control_images_as_references`` (image datasets), an item's control images
|
| 575 |
+
(``control_directory`` / ``control_path``) become its ordered image references for Ref2VA
|
| 576 |
+
instead, so Ref2VA requires one of the two. The target path is resolved the way the
|
| 577 |
+
dataset layer opens it.
|
| 578 |
+
"""
|
| 579 |
+
if task not in {"t2va", "fl2va", "ref2va"}:
|
| 580 |
+
raise ValueError(f"Unsupported MiniMax-H3 task: {task}")
|
| 581 |
+
if len(datasource) == 0:
|
| 582 |
+
raise ValueError("MiniMax-H3 dataset contains no items")
|
| 583 |
+
|
| 584 |
+
control_paths: Mapping[str, Sequence[str]] = {}
|
| 585 |
+
if control_images_as_references and task == "ref2va":
|
| 586 |
+
control_paths = datasource.get_control_paths()
|
| 587 |
+
if task == "ref2va" and not control_paths:
|
| 588 |
+
if not any("references" in datasource.get_item_extras(index).fields for index in range(len(datasource))):
|
| 589 |
+
raise ValueError(
|
| 590 |
+
"MiniMax-H3 Ref2VA requires per-item references: video_jsonl_file / image_jsonl_file records with"
|
| 591 |
+
" a references field, or (image datasets) control images as untimed references"
|
| 592 |
+
)
|
| 593 |
+
|
| 594 |
+
records = []
|
| 595 |
+
seen_targets: dict[Path, str] = {}
|
| 596 |
+
for index in range(len(datasource)):
|
| 597 |
+
target_path, caption = datasource.get_caption(index)
|
| 598 |
+
extras = datasource.get_item_extras(index)
|
| 599 |
+
context = f"H3 {extras.label}"
|
| 600 |
+
if not isinstance(target_path, str) or not target_path.strip():
|
| 601 |
+
raise ValueError(f"{context}: target path must be a non-empty path")
|
| 602 |
+
target = Path(target_path).expanduser().resolve()
|
| 603 |
+
if not target.is_file():
|
| 604 |
+
raise ValueError(f"{context}: target does not exist: {target}")
|
| 605 |
+
if target in seen_targets:
|
| 606 |
+
raise ValueError(f"{context}: duplicate target {target} (also {seen_targets[target]})")
|
| 607 |
+
seen_targets[target] = extras.label
|
| 608 |
+
records.append(
|
| 609 |
+
_record_from_fields(
|
| 610 |
+
target_path=target,
|
| 611 |
+
caption=caption,
|
| 612 |
+
fields=extras.fields,
|
| 613 |
+
base_directory=Path(extras.base_directory),
|
| 614 |
+
label=extras.label,
|
| 615 |
+
task=task,
|
| 616 |
+
probe=probe,
|
| 617 |
+
control_images=control_paths.get(target_path),
|
| 618 |
+
)
|
| 619 |
+
)
|
| 620 |
+
return records
|
third_party/musubi_tuner/minimax_h3/model.py
ADDED
|
@@ -0,0 +1,1298 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2025 The MiniMax Team and The HuggingFace Team. All rights reserved.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
#
|
| 15 |
+
# Adapted for Musubi from Hugging Face Diffusers PR #14355 at commit
|
| 16 |
+
# abc5e9bf71fd38f53cd471bc3acaa84bc5ecbfdc
|
| 17 |
+
# (models/transformers/transformer_minimax_h3.py).
|
| 18 |
+
# ComfyUI is used only as an independent numerical reference.
|
| 19 |
+
|
| 20 |
+
from __future__ import annotations
|
| 21 |
+
|
| 22 |
+
import json
|
| 23 |
+
import logging
|
| 24 |
+
import math
|
| 25 |
+
from collections import OrderedDict
|
| 26 |
+
from collections.abc import Mapping, Sequence
|
| 27 |
+
from dataclasses import dataclass, replace
|
| 28 |
+
from pathlib import Path
|
| 29 |
+
|
| 30 |
+
import torch
|
| 31 |
+
import torch.nn.functional as F
|
| 32 |
+
from torch import nn
|
| 33 |
+
|
| 34 |
+
from musubi_tuner.minimax_h3.checkpoint import (
|
| 35 |
+
load_safetensors_metadata,
|
| 36 |
+
resolve_safetensors_files,
|
| 37 |
+
)
|
| 38 |
+
from musubi_tuner.minimax_h3.media import TARGET_FPS
|
| 39 |
+
from musubi_tuner.minimax_h3.packing import (
|
| 40 |
+
AUDIO_CHANNELS,
|
| 41 |
+
VIDEO_CHANNELS,
|
| 42 |
+
VIDEO_PATCH_SIZE,
|
| 43 |
+
H3PackedLayout,
|
| 44 |
+
build_position_grid,
|
| 45 |
+
build_timestep_rows,
|
| 46 |
+
pack_audio_rows,
|
| 47 |
+
pack_video_rows,
|
| 48 |
+
unpack_targets,
|
| 49 |
+
)
|
| 50 |
+
from musubi_tuner.modules.attention import AttentionParams, attention
|
| 51 |
+
from musubi_tuner.modules.convrot_int8_utils import canonicalize_convrot_int8_key, has_comfy_quant_tensors
|
| 52 |
+
from musubi_tuner.modules.custom_offloading_utils import BlockSwapConfig, create_offloader
|
| 53 |
+
from musubi_tuner.utils.model_utils import create_cpu_offloading_wrapper
|
| 54 |
+
from musubi_tuner.utils.safetensors_utils import MemoryEfficientSafeOpen, WeightTransformHooks
|
| 55 |
+
|
| 56 |
+
logger = logging.getLogger(__name__)
|
| 57 |
+
|
| 58 |
+
_ROTARY_CACHE_SIZE = 2
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
@dataclass(frozen=True)
|
| 62 |
+
class MiniMaxH3Config:
|
| 63 |
+
in_channels: int = 24
|
| 64 |
+
audio_in_channels: int = 32
|
| 65 |
+
hidden_size: int = 5376
|
| 66 |
+
num_layers: int = 50
|
| 67 |
+
token_refiner_num_layers: int = 2
|
| 68 |
+
num_attention_heads: int = 56
|
| 69 |
+
attention_head_dim: int = 128
|
| 70 |
+
ffn_hidden_size: int = 14336
|
| 71 |
+
patch_size: tuple[int, int, int] = (1, 2, 2)
|
| 72 |
+
text_dim: int = 5120
|
| 73 |
+
timestep_input_dim: int = 256
|
| 74 |
+
time_embed_hidden_size: int = 5376
|
| 75 |
+
time_embed_dim: int = 2688
|
| 76 |
+
rope_inv_freq_len: int = 16
|
| 77 |
+
norm_eps: float = 1e-5
|
| 78 |
+
qk_norm_eps: float = 1e-5
|
| 79 |
+
final_norm_eps: float = 1e-5
|
| 80 |
+
# pruned-AdaLN artifacts: the sinusoidal time embedder is replaced by a published
|
| 81 |
+
# FP32 [adaln_curve_grid, time_embed_dim] lookup table interpolated over t in [0, 1]
|
| 82 |
+
adaln_curve_grid: int | None = None
|
| 83 |
+
# self-pruned AdaLN (--prune_adaln): the sinusoidal time embedder is retained and its
|
| 84 |
+
# SiLU'd output is projected onto a load-time mean-centered rank-r SVD basis; AdaLN
|
| 85 |
+
# projections take the r basis coefficients plus a constant channel carrying the mean
|
| 86 |
+
adaln_rank: int | None = None
|
| 87 |
+
|
| 88 |
+
def __post_init__(self) -> None:
|
| 89 |
+
integer_fields = (
|
| 90 |
+
"in_channels",
|
| 91 |
+
"audio_in_channels",
|
| 92 |
+
"hidden_size",
|
| 93 |
+
"num_layers",
|
| 94 |
+
"token_refiner_num_layers",
|
| 95 |
+
"num_attention_heads",
|
| 96 |
+
"attention_head_dim",
|
| 97 |
+
"ffn_hidden_size",
|
| 98 |
+
"text_dim",
|
| 99 |
+
"timestep_input_dim",
|
| 100 |
+
"time_embed_hidden_size",
|
| 101 |
+
"time_embed_dim",
|
| 102 |
+
"rope_inv_freq_len",
|
| 103 |
+
)
|
| 104 |
+
for field in integer_fields:
|
| 105 |
+
if getattr(self, field) <= 0:
|
| 106 |
+
raise ValueError(f"MiniMax-H3 {field} must be positive")
|
| 107 |
+
if self.in_channels != VIDEO_CHANNELS or self.audio_in_channels != AUDIO_CHANNELS:
|
| 108 |
+
raise ValueError("MiniMax-H3 R1 requires 24 video channels and 32 audio channels")
|
| 109 |
+
if self.patch_size != VIDEO_PATCH_SIZE:
|
| 110 |
+
raise ValueError(f"MiniMax-H3 R1 requires video patch size {VIDEO_PATCH_SIZE}")
|
| 111 |
+
if self.timestep_input_dim % 2:
|
| 112 |
+
raise ValueError("MiniMax-H3 timestep input width must be even")
|
| 113 |
+
if self.rope_inv_freq_len * 6 > self.attention_head_dim:
|
| 114 |
+
raise ValueError("MiniMax-H3 rotary width exceeds the attention head width")
|
| 115 |
+
if self.adaln_curve_grid is not None:
|
| 116 |
+
if self.adaln_rank is not None:
|
| 117 |
+
raise ValueError("MiniMax-H3 pruned AdaLN curve table and self-pruned basis are mutually exclusive")
|
| 118 |
+
if self.adaln_curve_grid != 1025:
|
| 119 |
+
raise ValueError("MiniMax-H3 pruned AdaLN requires exactly 1025 curve rows")
|
| 120 |
+
if self.time_embed_dim != 8:
|
| 121 |
+
raise ValueError("MiniMax-H3 pruned AdaLN requires time_embed_dim=8")
|
| 122 |
+
if self.adaln_rank is not None and self.adaln_rank <= 0:
|
| 123 |
+
raise ValueError("MiniMax-H3 self-pruned AdaLN rank must be positive")
|
| 124 |
+
|
| 125 |
+
@property
|
| 126 |
+
def is_pruned(self) -> bool:
|
| 127 |
+
return self.adaln_curve_grid is not None or self.adaln_rank is not None
|
| 128 |
+
|
| 129 |
+
@property
|
| 130 |
+
def adaln_in_features(self) -> int:
|
| 131 |
+
if self.adaln_rank is not None:
|
| 132 |
+
# r basis coefficients plus the constant channel carrying the curve mean
|
| 133 |
+
return self.adaln_rank + 1
|
| 134 |
+
return self.time_embed_dim
|
| 135 |
+
|
| 136 |
+
@property
|
| 137 |
+
def attention_inner_dim(self) -> int:
|
| 138 |
+
return self.num_attention_heads * self.attention_head_dim
|
| 139 |
+
|
| 140 |
+
@property
|
| 141 |
+
def video_patch_dim(self) -> int:
|
| 142 |
+
return self.in_channels * math.prod(self.patch_size)
|
| 143 |
+
|
| 144 |
+
@property
|
| 145 |
+
def block_adaln_out_features(self) -> int:
|
| 146 |
+
return 6 * self.hidden_size * 3
|
| 147 |
+
|
| 148 |
+
@property
|
| 149 |
+
def final_adaln_out_features(self) -> int:
|
| 150 |
+
return 2 * self.hidden_size
|
| 151 |
+
|
| 152 |
+
|
| 153 |
+
_PUBLISHED_CONFIG_FIELDS = {
|
| 154 |
+
"hidden_size": 5376,
|
| 155 |
+
"num_layers": 50,
|
| 156 |
+
"token_refiner_num_layers": 2,
|
| 157 |
+
"num_attention_heads": 56,
|
| 158 |
+
"attention_head_dim": 128,
|
| 159 |
+
"ffn_hidden_size": 14336,
|
| 160 |
+
"latents_dim": 24,
|
| 161 |
+
"audio_latents_dim": 32,
|
| 162 |
+
"patch_size": [1, 2, 2],
|
| 163 |
+
"text_dim": 5120,
|
| 164 |
+
"timestep_input_dim": 256,
|
| 165 |
+
"time_embed_hidden_size": 5376,
|
| 166 |
+
"time_embed_dim": 2688,
|
| 167 |
+
"adaln_out_features": 96768,
|
| 168 |
+
"final_adaln_out_features": 10752,
|
| 169 |
+
"rope_inv_freq_len": 16,
|
| 170 |
+
"norm_eps": 1e-5,
|
| 171 |
+
"qk_norm_eps": 1e-5,
|
| 172 |
+
"final_norm_eps": 1e-5,
|
| 173 |
+
"image_model": "minimax_h3",
|
| 174 |
+
}
|
| 175 |
+
|
| 176 |
+
|
| 177 |
+
def parse_h3_transformer_config(metadata: Mapping[str, str], *, allow_convrot_int8: bool = False) -> MiniMaxH3Config:
|
| 178 |
+
artifact_markers = " ".join(f"{key}={value}" for key, value in metadata.items() if key != "config").lower()
|
| 179 |
+
blocked_markers = ("fp8", "nvfp4") if allow_convrot_int8 else ("convrot", "int8", "fp8", "nvfp4", "quantized")
|
| 180 |
+
if any(marker in artifact_markers for marker in blocked_markers):
|
| 181 |
+
raise ValueError(
|
| 182 |
+
"Unsupported quantized MiniMax-H3 checkpoint. ConvRot INT8 checkpoints are detected from their"
|
| 183 |
+
" tensor structure (or pass --convrot_int8 to quantize a BF16 checkpoint); other quantized"
|
| 184 |
+
" formats (fp8/NVFP4) are not supported."
|
| 185 |
+
)
|
| 186 |
+
raw_config = metadata.get("config")
|
| 187 |
+
if raw_config is None:
|
| 188 |
+
raise ValueError("MiniMax-H3 transformer checkpoint is missing config metadata")
|
| 189 |
+
try:
|
| 190 |
+
root = json.loads(raw_config)
|
| 191 |
+
transformer = root["transformer"]
|
| 192 |
+
except (json.JSONDecodeError, KeyError, TypeError) as error:
|
| 193 |
+
raise ValueError("MiniMax-H3 transformer checkpoint has invalid config metadata") from error
|
| 194 |
+
if not isinstance(transformer, dict):
|
| 195 |
+
raise TypeError("MiniMax-H3 transformer config must be an object")
|
| 196 |
+
for field, expected in _PUBLISHED_CONFIG_FIELDS.items():
|
| 197 |
+
if field not in transformer:
|
| 198 |
+
raise ValueError(f"MiniMax-H3 transformer config is missing {field}")
|
| 199 |
+
actual = transformer[field]
|
| 200 |
+
if actual != expected:
|
| 201 |
+
raise ValueError(f"MiniMax-H3 transformer config {field} expected {expected}, got {actual}")
|
| 202 |
+
if transformer.get("adaln_curve_grid") is not None:
|
| 203 |
+
raise ValueError("Pruned MiniMax-H3 AdaLN must be classified from its tensor structure")
|
| 204 |
+
return MiniMaxH3Config()
|
| 205 |
+
|
| 206 |
+
|
| 207 |
+
def _read_h3_tensor_headers(files: Sequence[str | Path]) -> dict[str, tuple]:
|
| 208 |
+
"""Read tensor headers (dtype string, shape tuple) across shards, keyed by canonical name."""
|
| 209 |
+
headers: dict[str, tuple] = {}
|
| 210 |
+
for file in files:
|
| 211 |
+
checkpoint_path = str(Path(file).resolve())
|
| 212 |
+
with MemoryEfficientSafeOpen(checkpoint_path) as handle:
|
| 213 |
+
for raw_key in handle.keys():
|
| 214 |
+
key = canonicalize_convrot_int8_key(raw_key)
|
| 215 |
+
if key in headers:
|
| 216 |
+
raise ValueError(f"Duplicate MiniMax-H3 tensor header {key!r} in {checkpoint_path}")
|
| 217 |
+
raw_header = handle.header[raw_key]
|
| 218 |
+
headers[key] = (raw_header["dtype"], tuple(raw_header["shape"]))
|
| 219 |
+
return headers
|
| 220 |
+
|
| 221 |
+
|
| 222 |
+
def _published_pruned_h3_config() -> MiniMaxH3Config:
|
| 223 |
+
return MiniMaxH3Config(time_embed_dim=8, adaln_curve_grid=1025)
|
| 224 |
+
|
| 225 |
+
|
| 226 |
+
def _require_pruned_adaln_header(headers: Mapping[str, tuple], key: str, expected_shape: tuple) -> None:
|
| 227 |
+
header = headers.get(key)
|
| 228 |
+
if header is None:
|
| 229 |
+
raise ValueError(f"Pruned MiniMax-H3 checkpoint is missing {key}")
|
| 230 |
+
if header[1] != expected_shape:
|
| 231 |
+
raise ValueError(f"Pruned MiniMax-H3 {key} expected shape {expected_shape}, got {header[1]}")
|
| 232 |
+
|
| 233 |
+
|
| 234 |
+
def classify_h3_transformer(
|
| 235 |
+
files: Sequence[str | Path],
|
| 236 |
+
*,
|
| 237 |
+
allow_convrot_int8: bool = False,
|
| 238 |
+
) -> MiniMaxH3Config:
|
| 239 |
+
"""Classify a transformer checkpoint as the published full or pruned-AdaLN structure.
|
| 240 |
+
|
| 241 |
+
Pruned artifacts (published as ConvRot INT8 and as BF16) are recognized structurally:
|
| 242 |
+
the ``adaln_t_table`` FP32 [1025, 8] curve table replaces the sinusoidal time
|
| 243 |
+
embedder, and every AdaLN projection takes the 8-wide interpolated embedding
|
| 244 |
+
directly (no SiLU). Full checkpoints keep the published-config metadata validation.
|
| 245 |
+
"""
|
| 246 |
+
headers = _read_h3_tensor_headers(files)
|
| 247 |
+
table = headers.get("adaln_t_table")
|
| 248 |
+
if table is None:
|
| 249 |
+
return parse_h3_transformer_config(load_safetensors_metadata(files), allow_convrot_int8=allow_convrot_int8)
|
| 250 |
+
if any(key.startswith("time_embedder.") for key in headers):
|
| 251 |
+
raise ValueError("MiniMax-H3 checkpoint has mixed adaln_t_table and time_embedder structures")
|
| 252 |
+
table_dtype, table_shape = table
|
| 253 |
+
if table_dtype != "F32":
|
| 254 |
+
raise ValueError(f"Pruned MiniMax-H3 adaln_t_table must be F32, got {table_dtype}")
|
| 255 |
+
if table_shape != (1025, 8):
|
| 256 |
+
raise ValueError(f"Pruned MiniMax-H3 adaln_t_table must have shape [1025, 8], got {table_shape}")
|
| 257 |
+
config = _published_pruned_h3_config()
|
| 258 |
+
for index in range(config.num_layers):
|
| 259 |
+
prefix = f"blocks.{index}.adaln_proj.linear"
|
| 260 |
+
_require_pruned_adaln_header(headers, f"{prefix}.weight", (config.block_adaln_out_features, config.time_embed_dim))
|
| 261 |
+
_require_pruned_adaln_header(headers, f"{prefix}.bias", (config.block_adaln_out_features,))
|
| 262 |
+
_require_pruned_adaln_header(
|
| 263 |
+
headers, "final_layer.adaln_proj.linear.weight", (config.final_adaln_out_features, config.time_embed_dim)
|
| 264 |
+
)
|
| 265 |
+
_require_pruned_adaln_header(headers, "final_layer.adaln_proj.linear.bias", (config.final_adaln_out_features,))
|
| 266 |
+
return config
|
| 267 |
+
|
| 268 |
+
|
| 269 |
+
# --prune_adaln: rank 8 matches the published pruned artifacts; the grid samples the
|
| 270 |
+
# model time t in [0, 1] densely for the mean-centered SVD (the curve is smooth, so the
|
| 271 |
+
# basis is insensitive to the exact grid size beyond a few hundred points)
|
| 272 |
+
H3_PRUNE_ADALN_RANK = 8
|
| 273 |
+
H3_PRUNE_ADALN_GRID = 4096
|
| 274 |
+
|
| 275 |
+
_H3_ADALN_WEIGHT_SUFFIX = ".adaln_proj.linear.weight"
|
| 276 |
+
_H3_TIME_EMBEDDER_PREFIX = "time_embedder."
|
| 277 |
+
|
| 278 |
+
|
| 279 |
+
def read_h3_time_embedder_state(files: Sequence[str | Path]) -> dict[str, torch.Tensor]:
|
| 280 |
+
state: dict[str, torch.Tensor] = {}
|
| 281 |
+
for file in files:
|
| 282 |
+
with MemoryEfficientSafeOpen(str(Path(file).resolve())) as handle:
|
| 283 |
+
for key in handle.keys():
|
| 284 |
+
if key.startswith(_H3_TIME_EMBEDDER_PREFIX):
|
| 285 |
+
state[key] = handle.get_tensor(key)
|
| 286 |
+
return state
|
| 287 |
+
|
| 288 |
+
|
| 289 |
+
def compute_pruned_adaln_basis(
|
| 290 |
+
time_embedder_state: Mapping[str, torch.Tensor],
|
| 291 |
+
config: MiniMaxH3Config,
|
| 292 |
+
*,
|
| 293 |
+
rank: int = H3_PRUNE_ADALN_RANK,
|
| 294 |
+
grid: int = H3_PRUNE_ADALN_GRID,
|
| 295 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 296 |
+
"""Mean-centered rank-``rank`` SVD basis of the SiLU'd time-embedding curve.
|
| 297 |
+
|
| 298 |
+
Returns FP32 ``(basis [time_embed_dim, rank], mean [time_embed_dim])`` computed over
|
| 299 |
+
a uniform ``grid``-point sampling of the model time t in [0, 1]. The SiLU that the
|
| 300 |
+
full model applies per block is baked into the curve, so pruned projections consume
|
| 301 |
+
the coefficients directly.
|
| 302 |
+
"""
|
| 303 |
+
embedder = TimeEmbedder(config.timestep_input_dim, config.time_embed_hidden_size, config.time_embed_dim)
|
| 304 |
+
embedder.load_state_dict({key.removeprefix(_H3_TIME_EMBEDDER_PREFIX): value for key, value in time_embedder_state.items()})
|
| 305 |
+
with torch.no_grad():
|
| 306 |
+
curve = F.silu(embedder(torch.linspace(0.0, 1.0, grid, dtype=torch.float32)))
|
| 307 |
+
mean = curve.mean(dim=0)
|
| 308 |
+
_, _, v_rows = torch.linalg.svd(curve - mean, full_matrices=False)
|
| 309 |
+
return v_rows[:rank].T.contiguous(), mean
|
| 310 |
+
|
| 311 |
+
|
| 312 |
+
def build_prune_adaln_hooks(basis: torch.Tensor, mean: torch.Tensor) -> WeightTransformHooks:
|
| 313 |
+
"""Streaming rewrite W [out, D] -> W @ [basis | mean] [out, rank + 1] in BF16.
|
| 314 |
+
|
| 315 |
+
A pure per-tensor transform: biases are untouched because the constant coefficient
|
| 316 |
+
channel applies the mean term at runtime. The rank + 1 input width is not divisible
|
| 317 |
+
by any ConvRot group size, so --convrot_int8 skips the rewritten projections, the
|
| 318 |
+
same scope as the published pruned INT8 artifacts.
|
| 319 |
+
"""
|
| 320 |
+
projection = torch.cat((basis, mean[:, None]), dim=1)
|
| 321 |
+
|
| 322 |
+
def split_hook(key: str, tensor: torch.Tensor | None):
|
| 323 |
+
if not key.endswith(_H3_ADALN_WEIGHT_SUFFIX):
|
| 324 |
+
return None, None
|
| 325 |
+
if tensor is None:
|
| 326 |
+
return [key], None
|
| 327 |
+
pruned = (tensor.to(torch.float32) @ projection.to(tensor.device)).to(torch.bfloat16)
|
| 328 |
+
return [key], [pruned]
|
| 329 |
+
|
| 330 |
+
return WeightTransformHooks(split_hook=split_hook)
|
| 331 |
+
|
| 332 |
+
|
| 333 |
+
class TimeEmbedder(nn.Module):
|
| 334 |
+
def __init__(
|
| 335 |
+
self,
|
| 336 |
+
input_dim: int,
|
| 337 |
+
hidden_size: int,
|
| 338 |
+
output_dim: int,
|
| 339 |
+
*,
|
| 340 |
+
device: torch.device | str | None = None,
|
| 341 |
+
) -> None:
|
| 342 |
+
super().__init__()
|
| 343 |
+
self.input_dim = input_dim
|
| 344 |
+
self.proj_in = nn.Linear(input_dim, hidden_size, dtype=torch.float32, device=device)
|
| 345 |
+
self.proj_out = nn.Linear(hidden_size, output_dim, dtype=torch.float32, device=device)
|
| 346 |
+
|
| 347 |
+
def forward(self, timesteps: torch.Tensor) -> torch.Tensor:
|
| 348 |
+
half = self.input_dim // 2
|
| 349 |
+
frequencies = torch.exp(-math.log(10000.0) * torch.arange(half, dtype=torch.float32, device=timesteps.device) / half)
|
| 350 |
+
angles = timesteps.to(torch.float32)[:, None] * frequencies[None]
|
| 351 |
+
embedding = torch.cat((torch.cos(angles), torch.sin(angles)), dim=-1)
|
| 352 |
+
return self.proj_out(F.silu(self.proj_in(embedding)))
|
| 353 |
+
|
| 354 |
+
|
| 355 |
+
class AdalnProj(nn.Module):
|
| 356 |
+
def __init__(
|
| 357 |
+
self,
|
| 358 |
+
timestep_dim: int,
|
| 359 |
+
hidden_size: int,
|
| 360 |
+
expand: int,
|
| 361 |
+
modalities: int,
|
| 362 |
+
*,
|
| 363 |
+
dtype: torch.dtype,
|
| 364 |
+
apply_silu: bool = True,
|
| 365 |
+
device: torch.device | str | None = None,
|
| 366 |
+
) -> None:
|
| 367 |
+
super().__init__()
|
| 368 |
+
self.expand = expand
|
| 369 |
+
self.modalities = modalities
|
| 370 |
+
self.hidden_size = hidden_size
|
| 371 |
+
# pruned-AdaLN checkpoints bake the activation into the curve table, so the
|
| 372 |
+
# 8-wide interpolated embedding feeds the projection directly
|
| 373 |
+
self.apply_silu = apply_silu
|
| 374 |
+
self.linear = nn.Linear(
|
| 375 |
+
timestep_dim,
|
| 376 |
+
expand * hidden_size * modalities,
|
| 377 |
+
dtype=dtype,
|
| 378 |
+
device=device,
|
| 379 |
+
)
|
| 380 |
+
|
| 381 |
+
def forward(self, timestep_embeddings: torch.Tensor) -> tuple[torch.Tensor, ...]:
|
| 382 |
+
if self.apply_silu:
|
| 383 |
+
timestep_embeddings = F.silu(timestep_embeddings)
|
| 384 |
+
projected = self.linear(timestep_embeddings)
|
| 385 |
+
projected = projected.reshape(projected.shape[0] * self.modalities, self.expand * self.hidden_size)
|
| 386 |
+
return projected.chunk(self.expand, dim=-1)
|
| 387 |
+
|
| 388 |
+
|
| 389 |
+
def _apply_rope_split_half(hidden_states: torch.Tensor, rotation_table: torch.Tensor) -> torch.Tensor:
|
| 390 |
+
pairs = rotation_table.shape[-3]
|
| 391 |
+
rotary = torch.stack((hidden_states[..., :pairs], hidden_states[..., pairs : 2 * pairs]), dim=-1)
|
| 392 |
+
rotary = torch.matmul(rotation_table, rotary.unsqueeze(-1)).squeeze(-1)
|
| 393 |
+
rotary = torch.cat((rotary[..., 0], rotary[..., 1]), dim=-1)
|
| 394 |
+
return torch.cat((rotary, hidden_states[..., 2 * pairs :]), dim=-1)
|
| 395 |
+
|
| 396 |
+
|
| 397 |
+
def _element_extent(tensor: torch.Tensor) -> int:
|
| 398 |
+
"""Highest storage element index addressable through the view, plus one."""
|
| 399 |
+
return tensor.storage_offset() + sum((size - 1) * stride for size, stride in zip(tensor.shape, tensor.stride())) + 1
|
| 400 |
+
|
| 401 |
+
|
| 402 |
+
# torch SDPA's memory-efficient backend computes block base addresses in int32, so a view
|
| 403 |
+
# whose element offsets pass 2^31 silently reads wrapped memory (verified: with the fused-qkv
|
| 404 |
+
# value view, every row from the first 128-aligned block past the line is corrupt, turning
|
| 405 |
+
# the whole sample into noise). The released 15 s maximum at 1344x768 packs ~109k rows and
|
| 406 |
+
# crosses the line at row 99,968.
|
| 407 |
+
_INT32_SAFE_EXTENT = 2**31
|
| 408 |
+
|
| 409 |
+
|
| 410 |
+
class Attention(nn.Module):
|
| 411 |
+
def __init__(
|
| 412 |
+
self,
|
| 413 |
+
hidden_size: int,
|
| 414 |
+
num_heads: int,
|
| 415 |
+
head_dim: int,
|
| 416 |
+
qk_norm_eps: float,
|
| 417 |
+
*,
|
| 418 |
+
attn_mode: str,
|
| 419 |
+
split_attn: bool,
|
| 420 |
+
dtype: torch.dtype,
|
| 421 |
+
device: torch.device | str | None = None,
|
| 422 |
+
) -> None:
|
| 423 |
+
super().__init__()
|
| 424 |
+
self.num_heads = num_heads
|
| 425 |
+
self.head_dim = head_dim
|
| 426 |
+
self.inner_dim = num_heads * head_dim
|
| 427 |
+
self.attn_mode = attn_mode
|
| 428 |
+
self.split_attn = split_attn
|
| 429 |
+
self.qkv_proj = nn.Linear(hidden_size, self.inner_dim * 3, bias=False, dtype=dtype, device=device)
|
| 430 |
+
self.q_norm = nn.RMSNorm(head_dim, eps=qk_norm_eps, dtype=dtype, device=device)
|
| 431 |
+
self.k_norm = nn.RMSNorm(head_dim, eps=qk_norm_eps, dtype=dtype, device=device)
|
| 432 |
+
self.out_proj = nn.Linear(self.inner_dim, hidden_size, bias=False, dtype=dtype, device=device)
|
| 433 |
+
|
| 434 |
+
def forward(self, hidden_states: torch.Tensor, rotation_table: torch.Tensor | None = None) -> torch.Tensor:
|
| 435 |
+
batch_size, sequence_length, _ = hidden_states.shape
|
| 436 |
+
query, key, value = self.qkv_proj(hidden_states).split(self.inner_dim, dim=-1)
|
| 437 |
+
query = self.q_norm(query.reshape(batch_size, sequence_length, self.num_heads, self.head_dim))
|
| 438 |
+
key = self.k_norm(key.reshape(batch_size, sequence_length, self.num_heads, self.head_dim))
|
| 439 |
+
value = value.reshape(batch_size, sequence_length, self.num_heads, self.head_dim)
|
| 440 |
+
# query/key leave the RMSNorms as fresh contiguous tensors, but value is still a view
|
| 441 |
+
# into the fused qkv buffer (sequence stride 3 * inner_dim): materialize it before its
|
| 442 |
+
# offsets outgrow int32 address math in the SDPA backends
|
| 443 |
+
if _element_extent(value) > _INT32_SAFE_EXTENT:
|
| 444 |
+
value = value.contiguous()
|
| 445 |
+
if rotation_table is not None:
|
| 446 |
+
query = _apply_rope_split_half(query, rotation_table)
|
| 447 |
+
key = _apply_rope_split_half(key, rotation_table)
|
| 448 |
+
query = query.to(value)
|
| 449 |
+
key = key.to(value)
|
| 450 |
+
|
| 451 |
+
params = AttentionParams.create_attention_params(self.attn_mode, self.split_attn)
|
| 452 |
+
output = attention([query, key, value], attn_params=params)
|
| 453 |
+
return self.out_proj(output)
|
| 454 |
+
|
| 455 |
+
|
| 456 |
+
class MLP(nn.Module):
|
| 457 |
+
def __init__(
|
| 458 |
+
self,
|
| 459 |
+
hidden_size: int,
|
| 460 |
+
ffn_hidden_size: int,
|
| 461 |
+
*,
|
| 462 |
+
dtype: torch.dtype,
|
| 463 |
+
device: torch.device | str | None = None,
|
| 464 |
+
) -> None:
|
| 465 |
+
super().__init__()
|
| 466 |
+
self.fc1 = nn.Linear(hidden_size, ffn_hidden_size * 2, bias=False, dtype=dtype, device=device)
|
| 467 |
+
self.fc2 = nn.Linear(ffn_hidden_size, hidden_size, bias=False, dtype=dtype, device=device)
|
| 468 |
+
|
| 469 |
+
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
| 470 |
+
gate, values = self.fc1(hidden_states).chunk(2, dim=-1)
|
| 471 |
+
return self.fc2(F.silu(gate) * values)
|
| 472 |
+
|
| 473 |
+
|
| 474 |
+
class RefinerBlock(nn.Module):
|
| 475 |
+
def __init__(self, config: MiniMaxH3Config, *, attn_mode: str, split_attn: bool, dtype, device) -> None:
|
| 476 |
+
super().__init__()
|
| 477 |
+
self.norm1 = nn.RMSNorm(config.hidden_size, eps=config.norm_eps, dtype=dtype, device=device)
|
| 478 |
+
self.norm2 = nn.RMSNorm(config.hidden_size, eps=config.norm_eps, dtype=dtype, device=device)
|
| 479 |
+
self.attn = Attention(
|
| 480 |
+
config.hidden_size,
|
| 481 |
+
config.num_attention_heads,
|
| 482 |
+
config.attention_head_dim,
|
| 483 |
+
config.qk_norm_eps,
|
| 484 |
+
attn_mode=attn_mode,
|
| 485 |
+
split_attn=split_attn,
|
| 486 |
+
dtype=dtype,
|
| 487 |
+
device=device,
|
| 488 |
+
)
|
| 489 |
+
self.mlp = MLP(config.hidden_size, config.ffn_hidden_size, dtype=dtype, device=device)
|
| 490 |
+
|
| 491 |
+
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
| 492 |
+
hidden_states = hidden_states + self.attn(self.norm1(hidden_states))
|
| 493 |
+
return hidden_states + self.mlp(self.norm2(hidden_states))
|
| 494 |
+
|
| 495 |
+
|
| 496 |
+
class TokenRefiner(nn.Module):
|
| 497 |
+
def __init__(self, config: MiniMaxH3Config, *, attn_mode: str, split_attn: bool, dtype, device) -> None:
|
| 498 |
+
super().__init__()
|
| 499 |
+
self.blocks = nn.ModuleList(
|
| 500 |
+
[
|
| 501 |
+
RefinerBlock(config, attn_mode=attn_mode, split_attn=split_attn, dtype=dtype, device=device)
|
| 502 |
+
for _ in range(config.token_refiner_num_layers)
|
| 503 |
+
]
|
| 504 |
+
)
|
| 505 |
+
self.final_norm = nn.RMSNorm(config.hidden_size, eps=config.final_norm_eps, dtype=dtype, device=device)
|
| 506 |
+
|
| 507 |
+
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
| 508 |
+
for block in self.blocks:
|
| 509 |
+
hidden_states = block(hidden_states)
|
| 510 |
+
return self.final_norm(hidden_states)
|
| 511 |
+
|
| 512 |
+
|
| 513 |
+
def _mod_scale_shift(
|
| 514 |
+
hidden_states: torch.Tensor,
|
| 515 |
+
shift: torch.Tensor,
|
| 516 |
+
scale: torch.Tensor,
|
| 517 |
+
segments: tuple[tuple[int, int, int], ...],
|
| 518 |
+
) -> torch.Tensor:
|
| 519 |
+
# Callers pass a fresh norm output; these disjoint slice updates retain trainable AdaLN gradients.
|
| 520 |
+
for start, stop, row in segments:
|
| 521 |
+
hidden_states[:, start:stop].mul_(1.0 + scale[row]).add_(shift[row])
|
| 522 |
+
return hidden_states
|
| 523 |
+
|
| 524 |
+
|
| 525 |
+
def _mod_gate(
|
| 526 |
+
residual: torch.Tensor,
|
| 527 |
+
update: torch.Tensor,
|
| 528 |
+
gate: torch.Tensor,
|
| 529 |
+
segments: tuple[tuple[int, int, int], ...],
|
| 530 |
+
) -> torch.Tensor:
|
| 531 |
+
output = residual.clone()
|
| 532 |
+
for start, stop, row in segments:
|
| 533 |
+
output[:, start:stop].add_(update[:, start:stop] * gate[row])
|
| 534 |
+
return output
|
| 535 |
+
|
| 536 |
+
|
| 537 |
+
class DiTBlock(nn.Module):
|
| 538 |
+
def __init__(self, config: MiniMaxH3Config, *, attn_mode: str, split_attn: bool, dtype, device) -> None:
|
| 539 |
+
super().__init__()
|
| 540 |
+
self.norm1 = nn.RMSNorm(config.hidden_size, eps=config.norm_eps, dtype=dtype, device=device)
|
| 541 |
+
self.norm2 = nn.RMSNorm(config.hidden_size, eps=config.norm_eps, dtype=dtype, device=device)
|
| 542 |
+
self.attn = Attention(
|
| 543 |
+
config.hidden_size,
|
| 544 |
+
config.num_attention_heads,
|
| 545 |
+
config.attention_head_dim,
|
| 546 |
+
config.qk_norm_eps,
|
| 547 |
+
attn_mode=attn_mode,
|
| 548 |
+
split_attn=split_attn,
|
| 549 |
+
dtype=dtype,
|
| 550 |
+
device=device,
|
| 551 |
+
)
|
| 552 |
+
self.mlp = MLP(config.hidden_size, config.ffn_hidden_size, dtype=dtype, device=device)
|
| 553 |
+
self.adaln_proj = AdalnProj(
|
| 554 |
+
config.adaln_in_features,
|
| 555 |
+
config.hidden_size,
|
| 556 |
+
expand=6,
|
| 557 |
+
modalities=3,
|
| 558 |
+
dtype=dtype,
|
| 559 |
+
apply_silu=not config.is_pruned,
|
| 560 |
+
device=device,
|
| 561 |
+
)
|
| 562 |
+
|
| 563 |
+
def forward(
|
| 564 |
+
self,
|
| 565 |
+
hidden_states: torch.Tensor,
|
| 566 |
+
timestep_embeddings: torch.Tensor,
|
| 567 |
+
modulation_segments: tuple[tuple[int, int, int], ...],
|
| 568 |
+
rotation_table: torch.Tensor,
|
| 569 |
+
) -> torch.Tensor:
|
| 570 |
+
shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.adaln_proj(timestep_embeddings)
|
| 571 |
+
normalized = _mod_scale_shift(self.norm1(hidden_states), shift_msa, scale_msa, modulation_segments)
|
| 572 |
+
hidden_states = _mod_gate(
|
| 573 |
+
hidden_states,
|
| 574 |
+
self.attn(normalized, rotation_table),
|
| 575 |
+
gate_msa,
|
| 576 |
+
modulation_segments,
|
| 577 |
+
)
|
| 578 |
+
normalized = _mod_scale_shift(self.norm2(hidden_states), shift_mlp, scale_mlp, modulation_segments)
|
| 579 |
+
return _mod_gate(hidden_states, self.mlp(normalized), gate_mlp, modulation_segments)
|
| 580 |
+
|
| 581 |
+
|
| 582 |
+
class FinalLayer(nn.Module):
|
| 583 |
+
def __init__(
|
| 584 |
+
self,
|
| 585 |
+
hidden_size: int,
|
| 586 |
+
timestep_dim: int,
|
| 587 |
+
video_output_dim: int,
|
| 588 |
+
audio_output_dim: int,
|
| 589 |
+
*,
|
| 590 |
+
norm_eps: float = 1e-5,
|
| 591 |
+
dtype: torch.dtype,
|
| 592 |
+
apply_silu: bool = True,
|
| 593 |
+
device: torch.device | str | None = None,
|
| 594 |
+
) -> None:
|
| 595 |
+
super().__init__()
|
| 596 |
+
self.norm = nn.RMSNorm(hidden_size, eps=norm_eps, dtype=dtype, device=device)
|
| 597 |
+
self.adaln_proj = AdalnProj(
|
| 598 |
+
timestep_dim,
|
| 599 |
+
hidden_size,
|
| 600 |
+
expand=2,
|
| 601 |
+
modalities=1,
|
| 602 |
+
dtype=dtype,
|
| 603 |
+
apply_silu=apply_silu,
|
| 604 |
+
device=device,
|
| 605 |
+
)
|
| 606 |
+
self.video_out = nn.Linear(hidden_size, video_output_dim, dtype=torch.float32, device=device)
|
| 607 |
+
self.audio_out = nn.Linear(hidden_size, audio_output_dim, dtype=torch.float32, device=device)
|
| 608 |
+
|
| 609 |
+
def forward(
|
| 610 |
+
self,
|
| 611 |
+
hidden_states: torch.Tensor,
|
| 612 |
+
timestep_embeddings: torch.Tensor,
|
| 613 |
+
*,
|
| 614 |
+
video_slice: slice,
|
| 615 |
+
audio_slice: slice,
|
| 616 |
+
video_timestep_index: int,
|
| 617 |
+
audio_timestep_index: int,
|
| 618 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 619 |
+
shift, scale = self.adaln_proj(timestep_embeddings)
|
| 620 |
+
video = self.norm(hidden_states[:, video_slice])
|
| 621 |
+
video = video * (1.0 + scale[video_timestep_index]) + shift[video_timestep_index]
|
| 622 |
+
audio = self.norm(hidden_states[:, audio_slice])
|
| 623 |
+
audio = audio * (1.0 + scale[audio_timestep_index]) + shift[audio_timestep_index]
|
| 624 |
+
return self.video_out(video.to(torch.float32)), self.audio_out(audio.to(torch.float32))
|
| 625 |
+
|
| 626 |
+
|
| 627 |
+
@dataclass(frozen=True)
|
| 628 |
+
class MiniMaxH3Output:
|
| 629 |
+
video: torch.Tensor
|
| 630 |
+
audio: torch.Tensor
|
| 631 |
+
|
| 632 |
+
|
| 633 |
+
class MiniMaxH3Model(nn.Module):
|
| 634 |
+
def __init__(
|
| 635 |
+
self,
|
| 636 |
+
config: MiniMaxH3Config,
|
| 637 |
+
*,
|
| 638 |
+
attn_mode: str = "torch",
|
| 639 |
+
split_attn: bool = False,
|
| 640 |
+
dtype: torch.dtype = torch.bfloat16,
|
| 641 |
+
device: torch.device | str | None = None,
|
| 642 |
+
) -> None:
|
| 643 |
+
super().__init__()
|
| 644 |
+
if attn_mode not in {"torch", "sdpa", "flash", "flash3", "sageattn", "xformers"}:
|
| 645 |
+
raise ValueError(f"Unsupported MiniMax-H3 attention mode: {attn_mode}")
|
| 646 |
+
self.config = config
|
| 647 |
+
self.attn_mode = attn_mode
|
| 648 |
+
self.split_attn = split_attn
|
| 649 |
+
self.video_patch_proj = nn.Linear(config.video_patch_dim, config.hidden_size, dtype=torch.float32, device=device)
|
| 650 |
+
self.audio_patch_proj = nn.Linear(config.audio_in_channels, config.hidden_size, dtype=torch.float32, device=device)
|
| 651 |
+
self.condition_proj = nn.Linear(config.text_dim, config.hidden_size, dtype=dtype, device=device)
|
| 652 |
+
if config.adaln_curve_grid is not None:
|
| 653 |
+
self.time_embedder = None
|
| 654 |
+
self.register_buffer(
|
| 655 |
+
"adaln_t_table",
|
| 656 |
+
torch.empty(config.adaln_curve_grid, config.time_embed_dim, dtype=torch.float32, device=device),
|
| 657 |
+
)
|
| 658 |
+
else:
|
| 659 |
+
self.time_embedder = TimeEmbedder(
|
| 660 |
+
config.timestep_input_dim,
|
| 661 |
+
config.time_embed_hidden_size,
|
| 662 |
+
config.time_embed_dim,
|
| 663 |
+
device=device,
|
| 664 |
+
)
|
| 665 |
+
if config.adaln_rank is not None:
|
| 666 |
+
# load-time mean-centered SVD basis of the SiLU'd time-embedding curve
|
| 667 |
+
self.register_buffer(
|
| 668 |
+
"adaln_basis",
|
| 669 |
+
torch.empty(config.time_embed_dim, config.adaln_rank, dtype=torch.float32, device=device),
|
| 670 |
+
)
|
| 671 |
+
self.register_buffer(
|
| 672 |
+
"adaln_mean",
|
| 673 |
+
torch.empty(config.time_embed_dim, dtype=torch.float32, device=device),
|
| 674 |
+
)
|
| 675 |
+
self.rope = nn.Module()
|
| 676 |
+
inv_freq = torch.empty(config.rope_inv_freq_len, dtype=torch.float32, device=device)
|
| 677 |
+
self.rope.register_buffer("inv_freq", inv_freq)
|
| 678 |
+
self.token_refiner = TokenRefiner(
|
| 679 |
+
config,
|
| 680 |
+
attn_mode=attn_mode,
|
| 681 |
+
split_attn=split_attn,
|
| 682 |
+
dtype=dtype,
|
| 683 |
+
device=device,
|
| 684 |
+
)
|
| 685 |
+
self.blocks = nn.ModuleList(
|
| 686 |
+
[
|
| 687 |
+
DiTBlock(config, attn_mode=attn_mode, split_attn=split_attn, dtype=dtype, device=device)
|
| 688 |
+
for _ in range(config.num_layers)
|
| 689 |
+
]
|
| 690 |
+
)
|
| 691 |
+
self.final_layer = FinalLayer(
|
| 692 |
+
config.hidden_size,
|
| 693 |
+
config.adaln_in_features,
|
| 694 |
+
config.video_patch_dim,
|
| 695 |
+
config.audio_in_channels,
|
| 696 |
+
norm_eps=config.final_norm_eps,
|
| 697 |
+
dtype=dtype,
|
| 698 |
+
apply_silu=not config.is_pruned,
|
| 699 |
+
device=device,
|
| 700 |
+
)
|
| 701 |
+
|
| 702 |
+
self.gradient_checkpointing = False
|
| 703 |
+
self.activation_cpu_offloading = False
|
| 704 |
+
self.blocks_to_swap = 0
|
| 705 |
+
self.offloader = None
|
| 706 |
+
self._execution_device = torch.device(device) if device is not None else None
|
| 707 |
+
self._rotary_cache: OrderedDict[tuple[H3PackedLayout, torch.device, torch.dtype], torch.Tensor] = OrderedDict()
|
| 708 |
+
|
| 709 |
+
def _apply(self, fn, recurse: bool = True):
|
| 710 |
+
result = super()._apply(fn, recurse=recurse)
|
| 711 |
+
if hasattr(self, "_rotary_cache"):
|
| 712 |
+
self._rotary_cache.clear()
|
| 713 |
+
return result
|
| 714 |
+
|
| 715 |
+
def load_state_dict(self, state_dict, strict: bool = True, assign: bool = False):
|
| 716 |
+
self._rotary_cache.clear()
|
| 717 |
+
return super().load_state_dict(state_dict, strict=strict, assign=assign)
|
| 718 |
+
|
| 719 |
+
@property
|
| 720 |
+
def device(self) -> torch.device:
|
| 721 |
+
return self.video_patch_proj.weight.device
|
| 722 |
+
|
| 723 |
+
@property
|
| 724 |
+
def dtype(self) -> torch.dtype:
|
| 725 |
+
# norms are never quantized, so this sentinel stays the compute dtype even when a
|
| 726 |
+
# checkpoint quantizes more Linears than the published ConvRot INT8 scope
|
| 727 |
+
return self.blocks[0].norm1.weight.dtype
|
| 728 |
+
|
| 729 |
+
def _timestep_embeddings(self, unique_timesteps: torch.Tensor, execution_device: torch.device) -> torch.Tensor:
|
| 730 |
+
if unique_timesteps.ndim != 1:
|
| 731 |
+
raise ValueError("MiniMax-H3 unique timesteps must be a one-dimensional tensor")
|
| 732 |
+
if not self.config.is_pruned:
|
| 733 |
+
return self.time_embedder(unique_timesteps.to(execution_device)).to(self.dtype)
|
| 734 |
+
|
| 735 |
+
if self.config.adaln_rank is not None:
|
| 736 |
+
# self-pruned AdaLN: the exact FP32 time-embedding curve projected onto the
|
| 737 |
+
# load-time basis; the trailing constant channel applies the curve mean via
|
| 738 |
+
# the per-block projections (their weights are W @ [basis | mean])
|
| 739 |
+
embedding = self.time_embedder(unique_timesteps.to(execution_device))
|
| 740 |
+
centered = F.silu(embedding) - self.adaln_mean
|
| 741 |
+
coefficients = F.pad(centered @ self.adaln_basis, (0, 1), value=1.0)
|
| 742 |
+
return coefficients.to(self.dtype)
|
| 743 |
+
|
| 744 |
+
# pruned AdaLN: FP32 linear interpolation over the published [1025, 8] curve
|
| 745 |
+
# table; the model time t = 1 - sigma lives in [0, 1] and maps onto the grid
|
| 746 |
+
table = self.adaln_t_table
|
| 747 |
+
timesteps_fp32 = unique_timesteps.to(device=table.device, dtype=torch.float32)
|
| 748 |
+
position = timesteps_fp32.clamp(0.0, 1.0) * (table.shape[0] - 1)
|
| 749 |
+
lower = position.floor().long().clamp(max=table.shape[0] - 2)
|
| 750 |
+
fraction = position - lower.to(position.dtype)
|
| 751 |
+
embedding_fp32 = torch.lerp(table[lower], table[lower + 1], fraction[:, None])
|
| 752 |
+
return embedding_fp32.to(device=execution_device, dtype=self.dtype)
|
| 753 |
+
|
| 754 |
+
def enable_gradient_checkpointing(self, activation_cpu_offloading: bool = False) -> None:
|
| 755 |
+
self.gradient_checkpointing = True
|
| 756 |
+
self.activation_cpu_offloading = activation_cpu_offloading
|
| 757 |
+
|
| 758 |
+
def disable_gradient_checkpointing(self) -> None:
|
| 759 |
+
self.gradient_checkpointing = False
|
| 760 |
+
self.activation_cpu_offloading = False
|
| 761 |
+
|
| 762 |
+
def enable_block_swap(self, num_blocks: int, config: BlockSwapConfig) -> None:
|
| 763 |
+
if num_blocks <= 0:
|
| 764 |
+
raise ValueError("MiniMax-H3 blocks to swap must be positive")
|
| 765 |
+
if num_blocks > len(self.blocks) - 2:
|
| 766 |
+
raise ValueError(f"MiniMax-H3 cannot swap more than {len(self.blocks) - 2} of {len(self.blocks)} blocks")
|
| 767 |
+
self.blocks_to_swap = num_blocks
|
| 768 |
+
self._execution_device = torch.device(config.device)
|
| 769 |
+
self.offloader = create_offloader(
|
| 770 |
+
"minimax-h3",
|
| 771 |
+
self.blocks,
|
| 772 |
+
len(self.blocks),
|
| 773 |
+
num_blocks,
|
| 774 |
+
config,
|
| 775 |
+
)
|
| 776 |
+
|
| 777 |
+
def move_to_device_except_swap_blocks(self, device: torch.device) -> None:
|
| 778 |
+
device = torch.device(device)
|
| 779 |
+
if self.blocks_to_swap:
|
| 780 |
+
saved_blocks = self.blocks
|
| 781 |
+
self.blocks = nn.ModuleList()
|
| 782 |
+
self.to(device)
|
| 783 |
+
if self.blocks_to_swap:
|
| 784 |
+
self.blocks = saved_blocks
|
| 785 |
+
self._execution_device = device
|
| 786 |
+
|
| 787 |
+
def prepare_block_swap_before_forward(self) -> None:
|
| 788 |
+
if not self.blocks_to_swap:
|
| 789 |
+
return
|
| 790 |
+
if self.offloader is None:
|
| 791 |
+
raise RuntimeError("MiniMax-H3 block swap has no offloader")
|
| 792 |
+
self.offloader.prepare_block_devices_before_forward(self.blocks)
|
| 793 |
+
|
| 794 |
+
def switch_block_swap_for_inference(self) -> None:
|
| 795 |
+
if self.blocks_to_swap:
|
| 796 |
+
self.offloader.set_forward_only(True)
|
| 797 |
+
self.prepare_block_swap_before_forward()
|
| 798 |
+
|
| 799 |
+
def switch_block_swap_for_training(self) -> None:
|
| 800 |
+
if self.blocks_to_swap:
|
| 801 |
+
self.offloader.set_forward_only(False)
|
| 802 |
+
self.prepare_block_swap_before_forward()
|
| 803 |
+
|
| 804 |
+
def _assert_block_device(self, block: nn.Module, index: int) -> None:
|
| 805 |
+
expected = self._execution_device
|
| 806 |
+
if expected is None:
|
| 807 |
+
return
|
| 808 |
+
|
| 809 |
+
def matches(actual: torch.device) -> bool:
|
| 810 |
+
return actual.type == expected.type and (expected.index is None or actual.index == expected.index)
|
| 811 |
+
|
| 812 |
+
for name, parameter in block.named_parameters():
|
| 813 |
+
if not matches(parameter.device):
|
| 814 |
+
raise RuntimeError(
|
| 815 |
+
f"MiniMax-H3 block {index} parameter {name} is on {parameter.device}, expected {expected} after wait"
|
| 816 |
+
)
|
| 817 |
+
for name, buffer in block.named_buffers():
|
| 818 |
+
if not matches(buffer.device):
|
| 819 |
+
raise RuntimeError(f"MiniMax-H3 block {index} buffer {name} is on {buffer.device}, expected {expected} after wait")
|
| 820 |
+
|
| 821 |
+
def _rotation_table(
|
| 822 |
+
self,
|
| 823 |
+
position_ids: torch.Tensor,
|
| 824 |
+
dtype: torch.dtype,
|
| 825 |
+
*,
|
| 826 |
+
fine_position_ids: torch.Tensor | None = None,
|
| 827 |
+
fine_bands: int = 0,
|
| 828 |
+
) -> torch.Tensor:
|
| 829 |
+
positions = position_ids.to(device=self.rope.inv_freq.device, dtype=torch.float32)
|
| 830 |
+
if positions.ndim != 3 or positions.shape[-1] != 3:
|
| 831 |
+
raise ValueError("MiniMax-H3 position ids must be [B,S,3]")
|
| 832 |
+
per_axis = positions.unsqueeze(-1) * self.rope.inv_freq.reshape(1, 1, 1, -1)
|
| 833 |
+
temporal, height, width = per_axis.unbind(dim=2)
|
| 834 |
+
if fine_position_ids is not None and fine_bands:
|
| 835 |
+
# temporal-stretch experiment: the leading (sub-lattice) bands rotate by the
|
| 836 |
+
# unstretched grid so the per-token lattice phase stays as trained, while the
|
| 837 |
+
# remaining bands carry the stretched physical timeline
|
| 838 |
+
if fine_bands >= self.rope.inv_freq.shape[0]:
|
| 839 |
+
raise ValueError(
|
| 840 |
+
f"MiniMax-H3 temporal fine bands {fine_bands} must leave at least one of the"
|
| 841 |
+
f" {self.rope.inv_freq.shape[0]} rotary bands on the stretched clock"
|
| 842 |
+
)
|
| 843 |
+
fine_positions = fine_position_ids.to(device=self.rope.inv_freq.device, dtype=torch.float32)
|
| 844 |
+
if fine_positions.shape != positions.shape:
|
| 845 |
+
raise ValueError("MiniMax-H3 fine position ids must match the stretched grid shape")
|
| 846 |
+
fine_temporal = fine_positions[..., 0].unsqueeze(-1) * self.rope.inv_freq.reshape(1, 1, -1)
|
| 847 |
+
temporal = torch.cat((fine_temporal[..., :fine_bands], temporal[..., fine_bands:]), dim=-1)
|
| 848 |
+
half = torch.cat((temporal, height, width), dim=-1)
|
| 849 |
+
angles = torch.cat((half, half), dim=-1)
|
| 850 |
+
pair_count = angles.shape[-1] // 2
|
| 851 |
+
angles = angles[..., :pair_count]
|
| 852 |
+
cosine, sine = torch.cos(angles), torch.sin(angles)
|
| 853 |
+
return (
|
| 854 |
+
torch.stack((cosine, -sine, sine, cosine), dim=-1)
|
| 855 |
+
.reshape(
|
| 856 |
+
angles.shape[0],
|
| 857 |
+
angles.shape[1],
|
| 858 |
+
1,
|
| 859 |
+
pair_count,
|
| 860 |
+
2,
|
| 861 |
+
2,
|
| 862 |
+
)
|
| 863 |
+
.to(dtype)
|
| 864 |
+
)
|
| 865 |
+
|
| 866 |
+
def _cached_rotation_table(
|
| 867 |
+
self,
|
| 868 |
+
layout: H3PackedLayout,
|
| 869 |
+
*,
|
| 870 |
+
device: torch.device,
|
| 871 |
+
dtype: torch.dtype,
|
| 872 |
+
) -> torch.Tensor:
|
| 873 |
+
key = (layout, torch.device(device), dtype)
|
| 874 |
+
cached = self._rotary_cache.get(key)
|
| 875 |
+
if cached is not None:
|
| 876 |
+
self._rotary_cache.move_to_end(key)
|
| 877 |
+
return cached
|
| 878 |
+
position_ids = build_position_grid(layout, device=device)
|
| 879 |
+
fine_position_ids = None
|
| 880 |
+
if layout.temporal_fine_bands and layout.output_fps != TARGET_FPS:
|
| 881 |
+
fine_position_ids = build_position_grid(replace(layout, output_fps=TARGET_FPS, temporal_fine_bands=0), device=device)
|
| 882 |
+
rotation_table = self._rotation_table(
|
| 883 |
+
position_ids, dtype, fine_position_ids=fine_position_ids, fine_bands=layout.temporal_fine_bands
|
| 884 |
+
)
|
| 885 |
+
self._rotary_cache[key] = rotation_table
|
| 886 |
+
while len(self._rotary_cache) > _ROTARY_CACHE_SIZE:
|
| 887 |
+
self._rotary_cache.popitem(last=False)
|
| 888 |
+
return rotation_table
|
| 889 |
+
|
| 890 |
+
@staticmethod
|
| 891 |
+
def _validate_text_tags(tags: torch.Tensor, batch_size: int, text_length: int) -> torch.Tensor:
|
| 892 |
+
tags = torch.as_tensor(tags)
|
| 893 |
+
if tags.shape != (batch_size, text_length):
|
| 894 |
+
raise ValueError(f"MiniMax-H3 text token tags must be [{batch_size},{text_length}]")
|
| 895 |
+
return tags
|
| 896 |
+
|
| 897 |
+
@staticmethod
|
| 898 |
+
def _validate_condition_rows(
|
| 899 |
+
latents: Sequence[torch.Tensor],
|
| 900 |
+
segments,
|
| 901 |
+
pack,
|
| 902 |
+
*,
|
| 903 |
+
batch_size: int,
|
| 904 |
+
label: str,
|
| 905 |
+
) -> list[torch.Tensor]:
|
| 906 |
+
if len(latents) != len(segments):
|
| 907 |
+
raise ValueError(f"MiniMax-H3 expected {len(segments)} {label} tensors, got {len(latents)}")
|
| 908 |
+
rows = []
|
| 909 |
+
for tensor, segment in zip(latents, segments):
|
| 910 |
+
if tensor.shape[0] != batch_size:
|
| 911 |
+
raise ValueError(f"MiniMax-H3 {segment.role} batch size does not match the targets")
|
| 912 |
+
packed = pack(tensor)
|
| 913 |
+
if packed.shape[1] != segment.row_count:
|
| 914 |
+
raise ValueError(f"MiniMax-H3 {segment.role} has {packed.shape[1]} packed rows, expected {segment.row_count}")
|
| 915 |
+
rows.append(packed)
|
| 916 |
+
return rows
|
| 917 |
+
|
| 918 |
+
def forward(
|
| 919 |
+
self,
|
| 920 |
+
*,
|
| 921 |
+
video_latents: torch.Tensor,
|
| 922 |
+
audio_latents: torch.Tensor,
|
| 923 |
+
text_hidden_states: torch.Tensor,
|
| 924 |
+
text_token_tags: torch.Tensor,
|
| 925 |
+
layout: H3PackedLayout,
|
| 926 |
+
model_t_video: float | torch.Tensor,
|
| 927 |
+
model_t_audio: float | torch.Tensor,
|
| 928 |
+
visual_condition_latents: Sequence[torch.Tensor] = (),
|
| 929 |
+
audio_condition_latents: Sequence[torch.Tensor] = (),
|
| 930 |
+
visual_condition_clean: float = 0.999,
|
| 931 |
+
audio_condition_clean: float = 1.0,
|
| 932 |
+
) -> MiniMaxH3Output:
|
| 933 |
+
if video_latents.ndim != 5 or tuple(video_latents.shape[2:]) != (
|
| 934 |
+
layout.target_video.frames,
|
| 935 |
+
layout.target_video.height,
|
| 936 |
+
layout.target_video.width,
|
| 937 |
+
):
|
| 938 |
+
raise ValueError("MiniMax-H3 target video tensor does not match the packed layout")
|
| 939 |
+
if audio_latents.ndim != 4 or tuple(audio_latents.shape[1:]) != (
|
| 940 |
+
self.config.audio_in_channels,
|
| 941 |
+
2,
|
| 942 |
+
layout.target_audio_frames,
|
| 943 |
+
):
|
| 944 |
+
raise ValueError("MiniMax-H3 target audio tensor does not match the packed layout")
|
| 945 |
+
batch_size = video_latents.shape[0]
|
| 946 |
+
if batch_size != 1:
|
| 947 |
+
raise ValueError(f"MiniMax-H3 R1 requires batch_size=1, got {batch_size}; use gradient accumulation")
|
| 948 |
+
if audio_latents.shape[0] != batch_size:
|
| 949 |
+
raise ValueError("MiniMax-H3 target video and audio batch sizes differ")
|
| 950 |
+
if text_hidden_states.shape != (batch_size, layout.text_length, self.config.text_dim):
|
| 951 |
+
raise ValueError(f"MiniMax-H3 text hidden states must be [{batch_size},{layout.text_length},{self.config.text_dim}]")
|
| 952 |
+
text_token_tags = self._validate_text_tags(text_token_tags, batch_size, layout.text_length)
|
| 953 |
+
|
| 954 |
+
execution_device = self.video_patch_proj.weight.device
|
| 955 |
+
video_dtype = video_latents.dtype
|
| 956 |
+
audio_dtype = audio_latents.dtype
|
| 957 |
+
video_latents = video_latents.to(execution_device)
|
| 958 |
+
audio_latents = audio_latents.to(execution_device)
|
| 959 |
+
text_hidden_states = text_hidden_states.to(execution_device)
|
| 960 |
+
visual_condition_latents = tuple(tensor.to(execution_device) for tensor in visual_condition_latents)
|
| 961 |
+
audio_condition_latents = tuple(tensor.to(execution_device) for tensor in audio_condition_latents)
|
| 962 |
+
|
| 963 |
+
visual_segments = tuple(segment for segment in layout.segments if segment.kind == "visual_condition")
|
| 964 |
+
audio_segments = tuple(segment for segment in layout.segments if segment.kind == "audio_condition")
|
| 965 |
+
if layout.task == "fl2va":
|
| 966 |
+
expected_visual_geometries = layout.visual_conditions
|
| 967 |
+
elif layout.task == "ref2va":
|
| 968 |
+
expected_visual_geometries = tuple(
|
| 969 |
+
reference.video for reference in layout.references if reference.kind in {"image", "video"}
|
| 970 |
+
)
|
| 971 |
+
else:
|
| 972 |
+
expected_visual_geometries = ()
|
| 973 |
+
expected_audio_frames = (
|
| 974 |
+
tuple(reference.audio_frames for reference in layout.references if reference.audio_frames)
|
| 975 |
+
if layout.task == "ref2va"
|
| 976 |
+
else ()
|
| 977 |
+
)
|
| 978 |
+
for tensor, segment, geometry in zip(
|
| 979 |
+
visual_condition_latents,
|
| 980 |
+
visual_segments,
|
| 981 |
+
expected_visual_geometries,
|
| 982 |
+
):
|
| 983 |
+
actual = tuple(tensor.shape[2:]) if tensor.ndim == 5 else tuple(tensor.shape)
|
| 984 |
+
expected = (geometry.frames, geometry.height, geometry.width)
|
| 985 |
+
if actual != expected:
|
| 986 |
+
raise ValueError(
|
| 987 |
+
f"MiniMax-H3 {segment.role} geometry expected {'x'.join(map(str, expected))}, got {'x'.join(map(str, actual))}"
|
| 988 |
+
)
|
| 989 |
+
for tensor, segment, frames in zip(audio_condition_latents, audio_segments, expected_audio_frames):
|
| 990 |
+
actual = tuple(tensor.shape[1:]) if tensor.ndim == 4 else tuple(tensor.shape)
|
| 991 |
+
expected = (self.config.audio_in_channels, 2, frames)
|
| 992 |
+
if actual != expected:
|
| 993 |
+
raise ValueError(
|
| 994 |
+
f"MiniMax-H3 {segment.role} geometry expected {'x'.join(map(str, expected))}, got {'x'.join(map(str, actual))}"
|
| 995 |
+
)
|
| 996 |
+
visual_rows = self._validate_condition_rows(
|
| 997 |
+
visual_condition_latents,
|
| 998 |
+
visual_segments,
|
| 999 |
+
pack_video_rows,
|
| 1000 |
+
batch_size=batch_size,
|
| 1001 |
+
label="visual condition",
|
| 1002 |
+
)
|
| 1003 |
+
audio_rows = self._validate_condition_rows(
|
| 1004 |
+
audio_condition_latents,
|
| 1005 |
+
audio_segments,
|
| 1006 |
+
pack_audio_rows,
|
| 1007 |
+
batch_size=batch_size,
|
| 1008 |
+
label="audio condition",
|
| 1009 |
+
)
|
| 1010 |
+
visual_rows.append(pack_video_rows(video_latents))
|
| 1011 |
+
audio_rows.append(pack_audio_rows(audio_latents))
|
| 1012 |
+
projected_video = self.video_patch_proj(torch.cat(visual_rows, dim=1).to(torch.float32)).to(self.dtype)
|
| 1013 |
+
projected_audio = self.audio_patch_proj(torch.cat(audio_rows, dim=1).to(torch.float32)).to(self.dtype)
|
| 1014 |
+
|
| 1015 |
+
text = self.condition_proj(text_hidden_states.to(self.dtype))
|
| 1016 |
+
text = self.token_refiner(text)
|
| 1017 |
+
parts = []
|
| 1018 |
+
visual_offset = 0
|
| 1019 |
+
audio_offset = 0
|
| 1020 |
+
for segment in layout.segments:
|
| 1021 |
+
if segment.kind == "text":
|
| 1022 |
+
parts.append(text)
|
| 1023 |
+
elif segment.kind in {"visual_condition", "target_video"}:
|
| 1024 |
+
parts.append(projected_video[:, visual_offset : visual_offset + segment.row_count])
|
| 1025 |
+
visual_offset += segment.row_count
|
| 1026 |
+
else:
|
| 1027 |
+
parts.append(projected_audio[:, audio_offset : audio_offset + segment.row_count])
|
| 1028 |
+
audio_offset += segment.row_count
|
| 1029 |
+
hidden_states = torch.cat(parts, dim=1)
|
| 1030 |
+
if hidden_states.shape[1] != layout.row_count:
|
| 1031 |
+
raise RuntimeError("MiniMax-H3 packed residual stream length drifted from its layout")
|
| 1032 |
+
|
| 1033 |
+
timestep_rows = build_timestep_rows(
|
| 1034 |
+
layout,
|
| 1035 |
+
text_token_tags=text_token_tags,
|
| 1036 |
+
model_t_video=model_t_video,
|
| 1037 |
+
model_t_audio=model_t_audio,
|
| 1038 |
+
visual_condition_clean=visual_condition_clean,
|
| 1039 |
+
audio_condition_clean=audio_condition_clean,
|
| 1040 |
+
)
|
| 1041 |
+
timestep_embeddings = self._timestep_embeddings(timestep_rows.unique_timesteps, execution_device)
|
| 1042 |
+
rotation_table = self._cached_rotation_table(
|
| 1043 |
+
layout,
|
| 1044 |
+
device=execution_device,
|
| 1045 |
+
dtype=hidden_states.dtype,
|
| 1046 |
+
)
|
| 1047 |
+
|
| 1048 |
+
for index, block in enumerate(self.blocks):
|
| 1049 |
+
if self.blocks_to_swap:
|
| 1050 |
+
self.offloader.wait_for_block(index)
|
| 1051 |
+
self._assert_block_device(block, index)
|
| 1052 |
+
if self.gradient_checkpointing and self.training:
|
| 1053 |
+
forward_fn = block
|
| 1054 |
+
if self.activation_cpu_offloading:
|
| 1055 |
+
forward_fn = create_cpu_offloading_wrapper(forward_fn, execution_device)
|
| 1056 |
+
hidden_states = torch.utils.checkpoint.checkpoint(
|
| 1057 |
+
forward_fn,
|
| 1058 |
+
hidden_states,
|
| 1059 |
+
timestep_embeddings,
|
| 1060 |
+
timestep_rows.block_segments,
|
| 1061 |
+
rotation_table,
|
| 1062 |
+
use_reentrant=False,
|
| 1063 |
+
)
|
| 1064 |
+
else:
|
| 1065 |
+
hidden_states = block(
|
| 1066 |
+
hidden_states,
|
| 1067 |
+
timestep_embeddings,
|
| 1068 |
+
timestep_rows.block_segments,
|
| 1069 |
+
rotation_table,
|
| 1070 |
+
)
|
| 1071 |
+
if self.blocks_to_swap:
|
| 1072 |
+
self.offloader.submit_move_blocks_forward(self.blocks, index)
|
| 1073 |
+
|
| 1074 |
+
if hidden_states.device != execution_device:
|
| 1075 |
+
hidden_states = hidden_states.to(execution_device)
|
| 1076 |
+
|
| 1077 |
+
video_rows, audio_rows = self.final_layer(
|
| 1078 |
+
hidden_states,
|
| 1079 |
+
timestep_embeddings,
|
| 1080 |
+
video_slice=layout.target_video_segment.row_slice,
|
| 1081 |
+
audio_slice=layout.target_audio_segment.row_slice,
|
| 1082 |
+
video_timestep_index=timestep_rows.video_timestep_index,
|
| 1083 |
+
audio_timestep_index=timestep_rows.audio_timestep_index,
|
| 1084 |
+
)
|
| 1085 |
+
video, audio = unpack_targets(layout, video_rows, audio_rows)
|
| 1086 |
+
return MiniMaxH3Output(video=video.to(video_dtype), audio=audio.to(audio_dtype))
|
| 1087 |
+
|
| 1088 |
+
|
| 1089 |
+
# ConvRot INT8 scope mirrors the ComfyUI distribution: the five per-block Linears
|
| 1090 |
+
# (attn qkv/out, mlp fc1/fc2, adaln_proj). token_refiner blocks and the final layer
|
| 1091 |
+
# stay BF16. adaln_proj's in_features (2688) is not a multiple of 256, so the group
|
| 1092 |
+
# size falls back to 64 there — the same choice ComfyUI publishes in `comfy_quant`.
|
| 1093 |
+
H3_CONVROT_INT8_TARGET_KEYS = ["attn.qkv_proj", "attn.out_proj", "mlp.fc1", "mlp.fc2", "adaln_proj.linear"]
|
| 1094 |
+
H3_CONVROT_INT8_EXCLUDE_KEYS = ["token_refiner.", "final_layer."]
|
| 1095 |
+
H3_CONVROT_INT8_ALLOWED_GROUPSIZES = (256, 64)
|
| 1096 |
+
|
| 1097 |
+
|
| 1098 |
+
def _load_h3_transformer_convrot_int8(
|
| 1099 |
+
files: list[Path],
|
| 1100 |
+
config: MiniMaxH3Config,
|
| 1101 |
+
*,
|
| 1102 |
+
device: torch.device,
|
| 1103 |
+
quant_device: torch.device,
|
| 1104 |
+
bwd_mode: str,
|
| 1105 |
+
attn_mode: str,
|
| 1106 |
+
split_attn: bool,
|
| 1107 |
+
disable_numpy_memmap: bool,
|
| 1108 |
+
lora_weights: list[dict] | None = None,
|
| 1109 |
+
lora_multipliers: list[float] | None = None,
|
| 1110 |
+
prune_hooks: WeightTransformHooks | None = None,
|
| 1111 |
+
prune_state: dict[str, torch.Tensor] | None = None,
|
| 1112 |
+
) -> MiniMaxH3Model:
|
| 1113 |
+
"""Load the transformer with ConvRot INT8 base weights.
|
| 1114 |
+
|
| 1115 |
+
BF16 checkpoints are quantized on the fly on ``quant_device`` (LoRA weights, if any,
|
| 1116 |
+
are merged into the BF16 weights first); ComfyUI pre-quantized checkpoints are
|
| 1117 |
+
converted to the Musubi layout during the same streaming pass. ``device`` is where
|
| 1118 |
+
the weights end up ("cpu" under block swap). ``prune_hooks`` (--prune_adaln) rewrites
|
| 1119 |
+
the AdaLN projections before the quantization decision, which then skips them.
|
| 1120 |
+
"""
|
| 1121 |
+
from accelerate import init_empty_weights
|
| 1122 |
+
|
| 1123 |
+
from musubi_tuner.modules.convrot_int8_utils import ConvRotInt8Quantizer, apply_convrot_int8_monkey_patch
|
| 1124 |
+
from musubi_tuner.utils.lora_utils import load_safetensors_with_lora_and_fp8
|
| 1125 |
+
|
| 1126 |
+
with init_empty_weights():
|
| 1127 |
+
model = MiniMaxH3Model(config, attn_mode=attn_mode, split_attn=split_attn, dtype=torch.bfloat16)
|
| 1128 |
+
|
| 1129 |
+
quantizer = ConvRotInt8Quantizer(
|
| 1130 |
+
H3_CONVROT_INT8_TARGET_KEYS,
|
| 1131 |
+
H3_CONVROT_INT8_EXCLUDE_KEYS,
|
| 1132 |
+
allowed_groupsizes=H3_CONVROT_INT8_ALLOWED_GROUPSIZES,
|
| 1133 |
+
)
|
| 1134 |
+
sd = load_safetensors_with_lora_and_fp8(
|
| 1135 |
+
model_files=[str(path) for path in files],
|
| 1136 |
+
lora_weights_list=lora_weights,
|
| 1137 |
+
lora_multipliers=lora_multipliers,
|
| 1138 |
+
fp8_optimization=False,
|
| 1139 |
+
calc_device=quant_device,
|
| 1140 |
+
move_to_device=(device == quant_device),
|
| 1141 |
+
dit_weight_dtype=None,
|
| 1142 |
+
disable_numpy_memmap=disable_numpy_memmap,
|
| 1143 |
+
weight_transform_hooks=prune_hooks,
|
| 1144 |
+
quantizer=quantizer,
|
| 1145 |
+
)
|
| 1146 |
+
if prune_state is not None:
|
| 1147 |
+
sd.update(prune_state)
|
| 1148 |
+
apply_convrot_int8_monkey_patch(model, sd, bwd_mode=bwd_mode, groupsize_map=quantizer.module_groupsizes)
|
| 1149 |
+
# int8 tensors cannot require grad, and load_state_dict(assign=True) re-wraps incoming
|
| 1150 |
+
# tensors with the meta params' requires_grad (default True); the base is frozen anyway.
|
| 1151 |
+
model.requires_grad_(False)
|
| 1152 |
+
for key in sd.keys():
|
| 1153 |
+
# the H3 DiT stores only BF16 compute weights and FP32 fixed-precision islands;
|
| 1154 |
+
# F16 tensors (e.g. pruned-artifact AdaLN projections) convert to the compute dtype
|
| 1155 |
+
if sd[key].dtype == torch.float16:
|
| 1156 |
+
sd[key] = sd[key].to(torch.bfloat16)
|
| 1157 |
+
if device.type != "cpu":
|
| 1158 |
+
sd[key] = sd[key].to(device)
|
| 1159 |
+
# strict load keeps the R1 key verification: missing/unexpected keys raise
|
| 1160 |
+
model.load_state_dict(sd, strict=True, assign=True)
|
| 1161 |
+
model.eval()
|
| 1162 |
+
return model
|
| 1163 |
+
|
| 1164 |
+
|
| 1165 |
+
def _load_h3_transformer_bf16(
|
| 1166 |
+
files: Sequence[str | Path],
|
| 1167 |
+
config: MiniMaxH3Config,
|
| 1168 |
+
*,
|
| 1169 |
+
device: torch.device | str,
|
| 1170 |
+
attn_mode: str,
|
| 1171 |
+
split_attn: bool,
|
| 1172 |
+
disable_numpy_memmap: bool,
|
| 1173 |
+
prune_hooks: WeightTransformHooks | None = None,
|
| 1174 |
+
prune_state: dict[str, torch.Tensor] | None = None,
|
| 1175 |
+
) -> MiniMaxH3Model:
|
| 1176 |
+
from accelerate import init_empty_weights
|
| 1177 |
+
|
| 1178 |
+
from musubi_tuner.utils.safetensors_utils import load_safetensors
|
| 1179 |
+
|
| 1180 |
+
with init_empty_weights():
|
| 1181 |
+
model = MiniMaxH3Model(config, attn_mode=attn_mode, split_attn=split_attn, dtype=torch.bfloat16)
|
| 1182 |
+
|
| 1183 |
+
# load straight to the target device: on CUDA the per-tensor memmap path avoids a
|
| 1184 |
+
# resident full-model CPU copy (pages stay file-backed) and the .to(device) below
|
| 1185 |
+
# becomes a no-op; under block swap the caller passes "cpu" and nothing changes
|
| 1186 |
+
device = torch.device(device)
|
| 1187 |
+
sd: dict[str, torch.Tensor] = {}
|
| 1188 |
+
if prune_hooks is not None:
|
| 1189 |
+
# streaming per-tensor load so each full [out, D] AdaLN weight is rewritten to
|
| 1190 |
+
# [out, rank + 1] as it arrives; peak memory stays near the pruned model size
|
| 1191 |
+
from musubi_tuner.utils.lora_utils import load_safetensors_with_fp8_optimization_and_hook
|
| 1192 |
+
|
| 1193 |
+
sd = load_safetensors_with_fp8_optimization_and_hook(
|
| 1194 |
+
[str(file) for file in files],
|
| 1195 |
+
False,
|
| 1196 |
+
device,
|
| 1197 |
+
move_to_device=True,
|
| 1198 |
+
weight_transform_hooks=prune_hooks,
|
| 1199 |
+
disable_numpy_memmap=disable_numpy_memmap,
|
| 1200 |
+
)
|
| 1201 |
+
sd.update(prune_state or {})
|
| 1202 |
+
else:
|
| 1203 |
+
for file in files:
|
| 1204 |
+
sd.update(load_safetensors(str(file), device=device, disable_mmap=True, disable_numpy_memmap=disable_numpy_memmap))
|
| 1205 |
+
# pruned artifacts publish the AdaLN projections in F16; convert them to the BF16
|
| 1206 |
+
# compute dtype (the same treatment as the ConvRot INT8 loader)
|
| 1207 |
+
if config.is_pruned:
|
| 1208 |
+
for key, tensor in sd.items():
|
| 1209 |
+
if ".adaln_proj.linear." in key and tensor.dtype == torch.float16:
|
| 1210 |
+
sd[key] = tensor.to(torch.bfloat16)
|
| 1211 |
+
# the model definition is the dtype source of truth: BF16 compute weights plus FP32
|
| 1212 |
+
# fixed-precision islands must arrive exactly as published, without silent conversion
|
| 1213 |
+
expected_state = model.state_dict()
|
| 1214 |
+
dtype_mismatches = sorted(
|
| 1215 |
+
f"{key}: expected {expected_state[key].dtype}, got {tensor.dtype}"
|
| 1216 |
+
for key, tensor in sd.items()
|
| 1217 |
+
if key in expected_state and tensor.dtype != expected_state[key].dtype
|
| 1218 |
+
)
|
| 1219 |
+
if dtype_mismatches:
|
| 1220 |
+
raise ValueError(f"MiniMax-H3 checkpoint dtype mismatch: {dtype_mismatches[:20]}")
|
| 1221 |
+
model.load_state_dict(sd, strict=True, assign=True)
|
| 1222 |
+
model.to(device)
|
| 1223 |
+
model.eval()
|
| 1224 |
+
return model
|
| 1225 |
+
|
| 1226 |
+
|
| 1227 |
+
def load_h3_transformer(
|
| 1228 |
+
checkpoint_path: str | Path,
|
| 1229 |
+
*,
|
| 1230 |
+
device: torch.device | str,
|
| 1231 |
+
dtype: torch.dtype = torch.bfloat16,
|
| 1232 |
+
attn_mode: str = "torch",
|
| 1233 |
+
split_attn: bool = False,
|
| 1234 |
+
disable_numpy_memmap: bool = False,
|
| 1235 |
+
convrot_int8: bool = False,
|
| 1236 |
+
convrot_int8_bwd: str = "bf16",
|
| 1237 |
+
quant_device: torch.device | str | None = None,
|
| 1238 |
+
lora_weights: list[dict] | None = None,
|
| 1239 |
+
lora_multipliers: list[float] | None = None,
|
| 1240 |
+
prune_adaln: bool = False,
|
| 1241 |
+
) -> MiniMaxH3Model:
|
| 1242 |
+
if dtype != torch.bfloat16:
|
| 1243 |
+
raise ValueError("MiniMax-H3 accepts only BF16 transformer checkpoints")
|
| 1244 |
+
files = resolve_safetensors_files(checkpoint_path)
|
| 1245 |
+
# Pre-quantized ConvRot INT8 artifacts (full or pruned) and pruned BF16 artifacts are
|
| 1246 |
+
# detected from their tensor structure; --convrot_int8 additionally quantizes BF16
|
| 1247 |
+
# checkpoints (full or pruned) on the fly — pruned AdaLN projections are 8-wide and
|
| 1248 |
+
# fall outside every ConvRot group size, so the quantizer skips them automatically.
|
| 1249 |
+
prequantized = has_comfy_quant_tensors(files, disable_numpy_memmap=disable_numpy_memmap)
|
| 1250 |
+
use_convrot_int8 = convrot_int8 or prequantized
|
| 1251 |
+
config = classify_h3_transformer(files, allow_convrot_int8=use_convrot_int8)
|
| 1252 |
+
prune_hooks: WeightTransformHooks | None = None
|
| 1253 |
+
prune_state: dict[str, torch.Tensor] | None = None
|
| 1254 |
+
if prune_adaln:
|
| 1255 |
+
if config.is_pruned:
|
| 1256 |
+
logger.info("MiniMax-H3 checkpoint is already pruned; --prune_adaln has no effect")
|
| 1257 |
+
elif prequantized:
|
| 1258 |
+
raise ValueError(
|
| 1259 |
+
"--prune_adaln requires a BF16 MiniMax-H3 checkpoint; pre-quantized ConvRot INT8 checkpoints"
|
| 1260 |
+
" cannot be pruned at load time (use the published pruned artifact instead)"
|
| 1261 |
+
)
|
| 1262 |
+
else:
|
| 1263 |
+
config = replace(config, adaln_rank=H3_PRUNE_ADALN_RANK)
|
| 1264 |
+
basis, mean = compute_pruned_adaln_basis(read_h3_time_embedder_state(files), config)
|
| 1265 |
+
prune_hooks = build_prune_adaln_hooks(basis, mean)
|
| 1266 |
+
prune_state = {"adaln_basis": basis, "adaln_mean": mean}
|
| 1267 |
+
logger.info(
|
| 1268 |
+
f"Pruning MiniMax-H3 AdaLN projections at load time (mean-centered rank-{H3_PRUNE_ADALN_RANK} basis,"
|
| 1269 |
+
" time embedder retained)"
|
| 1270 |
+
)
|
| 1271 |
+
if use_convrot_int8:
|
| 1272 |
+
device = torch.device(device)
|
| 1273 |
+
return _load_h3_transformer_convrot_int8(
|
| 1274 |
+
[Path(checkpoint_path)], # unexpanded: the streaming loader expands split shards itself
|
| 1275 |
+
config,
|
| 1276 |
+
device=device,
|
| 1277 |
+
quant_device=device if quant_device is None else torch.device(quant_device),
|
| 1278 |
+
bwd_mode=convrot_int8_bwd,
|
| 1279 |
+
attn_mode=attn_mode,
|
| 1280 |
+
split_attn=split_attn,
|
| 1281 |
+
disable_numpy_memmap=disable_numpy_memmap,
|
| 1282 |
+
lora_weights=lora_weights,
|
| 1283 |
+
lora_multipliers=lora_multipliers,
|
| 1284 |
+
prune_hooks=prune_hooks,
|
| 1285 |
+
prune_state=prune_state,
|
| 1286 |
+
)
|
| 1287 |
+
if lora_weights:
|
| 1288 |
+
raise ValueError("MiniMax-H3 load-time LoRA merge is only wired for the ConvRot INT8 path")
|
| 1289 |
+
return _load_h3_transformer_bf16(
|
| 1290 |
+
files,
|
| 1291 |
+
config,
|
| 1292 |
+
device=device,
|
| 1293 |
+
attn_mode=attn_mode,
|
| 1294 |
+
split_attn=split_attn,
|
| 1295 |
+
disable_numpy_memmap=disable_numpy_memmap,
|
| 1296 |
+
prune_hooks=prune_hooks,
|
| 1297 |
+
prune_state=prune_state,
|
| 1298 |
+
)
|
third_party/musubi_tuner/minimax_h3/packing.py
ADDED
|
@@ -0,0 +1,711 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2026 The MiniMax and HuggingFace Teams. All rights reserved.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
#
|
| 15 |
+
# Adapted for Musubi from Hugging Face Diffusers PR #14355 at commit
|
| 16 |
+
# abc5e9bf71fd38f53cd471bc3acaa84bc5ecbfdc
|
| 17 |
+
# (modular_pipelines/minimax_h3/packing.py and packing_ref2va.py).
|
| 18 |
+
# ComfyUI is used only as an independent numerical reference.
|
| 19 |
+
|
| 20 |
+
from __future__ import annotations
|
| 21 |
+
|
| 22 |
+
import math
|
| 23 |
+
import re
|
| 24 |
+
from collections.abc import Sequence
|
| 25 |
+
from dataclasses import dataclass
|
| 26 |
+
from typing import Literal
|
| 27 |
+
|
| 28 |
+
import torch
|
| 29 |
+
|
| 30 |
+
from musubi_tuner.minimax_h3.media import TARGET_FPS, H3Task, audio_latent_frames
|
| 31 |
+
|
| 32 |
+
VIDEO_CHANNELS = 24
|
| 33 |
+
AUDIO_CHANNELS = 32
|
| 34 |
+
STEREO_CHANNELS = 2
|
| 35 |
+
VIDEO_PATCH_SIZE = (1, 2, 2)
|
| 36 |
+
FRAME_PER_TOKEN = (1, 4, 4, 4, 4)
|
| 37 |
+
FRAME_RESCALE = 5.0 / 3.0
|
| 38 |
+
|
| 39 |
+
H3ReferenceKind = Literal["image", "audio", "video"]
|
| 40 |
+
H3SegmentKind = Literal["text", "visual_condition", "audio_condition", "target_audio", "target_video"]
|
| 41 |
+
H3ConditionFamily = Literal["fl", "one_frame", "reference"]
|
| 42 |
+
|
| 43 |
+
# Condition role vocabulary. A role names one condition block of a layout and, prefixed with
|
| 44 |
+
# `latents_`, the cache/batch entry that carries its latent (see dataset/cache_io.py):
|
| 45 |
+
# - the released video FL2VA anchors `first` / `last` (the name selects the anchor time)
|
| 46 |
+
# - the ordered one-frame FL2VA slots `cond_{i:03d}` (times come from the control indices)
|
| 47 |
+
# - the numbered Ref2VA reference rows `ref_{i:03d}_{image|video|audio}`
|
| 48 |
+
FL_CONDITION_ROLES: tuple[str, ...] = ("first", "last")
|
| 49 |
+
REFERENCE_KINDS: tuple[H3ReferenceKind, ...] = ("image", "video", "audio")
|
| 50 |
+
_ONE_FRAME_CONDITION_ROLE = re.compile(r"^cond_(\d{3})$")
|
| 51 |
+
_REFERENCE_CONDITION_ROLE = re.compile(r"^ref_(\d{3})_(image|video|audio)$")
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
@dataclass(frozen=True)
|
| 55 |
+
class H3VideoGeometry:
|
| 56 |
+
frames: int
|
| 57 |
+
height: int
|
| 58 |
+
width: int
|
| 59 |
+
|
| 60 |
+
def __post_init__(self) -> None:
|
| 61 |
+
if self.frames <= 0:
|
| 62 |
+
raise ValueError(f"MiniMax-H3 video geometry needs positive frames, got {self.frames}")
|
| 63 |
+
if self.height <= 0 or self.width <= 0 or self.height % 2 or self.width % 2:
|
| 64 |
+
raise ValueError(f"MiniMax-H3 latent video axes must be positive multiples of 2, got {self.height}x{self.width}")
|
| 65 |
+
|
| 66 |
+
@property
|
| 67 |
+
def frame_rows(self) -> int:
|
| 68 |
+
return (self.height // 2) * (self.width // 2)
|
| 69 |
+
|
| 70 |
+
@property
|
| 71 |
+
def row_count(self) -> int:
|
| 72 |
+
return self.frames * self.frame_rows
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
@dataclass(frozen=True)
|
| 76 |
+
class H3ReferenceGeometry:
|
| 77 |
+
kind: H3ReferenceKind
|
| 78 |
+
video: H3VideoGeometry | None = None
|
| 79 |
+
audio_frames: int = 0
|
| 80 |
+
|
| 81 |
+
def __post_init__(self) -> None:
|
| 82 |
+
if self.kind not in {"image", "audio", "video"}:
|
| 83 |
+
raise ValueError(f"Unsupported MiniMax-H3 reference kind: {self.kind}")
|
| 84 |
+
if self.audio_frames < 0:
|
| 85 |
+
raise ValueError(f"MiniMax-H3 reference audio length must be nonnegative, got {self.audio_frames}")
|
| 86 |
+
if self.kind == "image":
|
| 87 |
+
if self.video is None or self.video.frames != 1 or self.audio_frames:
|
| 88 |
+
raise ValueError("MiniMax-H3 image references require one visual frame and no audio rows")
|
| 89 |
+
elif self.kind == "audio":
|
| 90 |
+
if self.video is not None or self.audio_frames <= 0:
|
| 91 |
+
raise ValueError("MiniMax-H3 audio references require audio rows and no visual geometry")
|
| 92 |
+
else:
|
| 93 |
+
if self.video is None:
|
| 94 |
+
raise ValueError("MiniMax-H3 video references require visual geometry")
|
| 95 |
+
_validate_video_latent_frames(self.video.frames, "reference video")
|
| 96 |
+
if self.audio_frames:
|
| 97 |
+
expected = _expected_audio_frames(self.video.frames)
|
| 98 |
+
if self.audio_frames != expected:
|
| 99 |
+
raise ValueError(f"MiniMax-H3 reference video audio has {self.audio_frames} frames, expected {expected}")
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
def one_frame_condition_role(index: int) -> str:
|
| 103 |
+
"""Role of the index-th one-frame FL2VA visual condition (``cond_000``, ``cond_001``, ...).
|
| 104 |
+
|
| 105 |
+
One-frame conditions are placed by explicit time overrides, so unlike the released video
|
| 106 |
+
FL2VA roles (``first``/``last``, whose names select the anchor time) they are plain ordered
|
| 107 |
+
slots, any number of them; the text presentation numbers ``<Picture i>`` in the same order.
|
| 108 |
+
"""
|
| 109 |
+
if index < 0:
|
| 110 |
+
raise ValueError(f"MiniMax-H3 one-frame condition index must be nonnegative, got {index}")
|
| 111 |
+
return f"cond_{index:03d}"
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def one_frame_condition_roles(count: int) -> tuple[str, ...]:
|
| 115 |
+
return tuple(one_frame_condition_role(index) for index in range(count))
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
def reference_condition_role(index: int, kind: H3ReferenceKind) -> str:
|
| 119 |
+
"""Role of the index-th Ref2VA reference's ``kind`` rows (``ref_000_image``, ``ref_001_audio``, ...).
|
| 120 |
+
|
| 121 |
+
A video reference with audio owns two roles (``_video`` and ``_audio``) under one index.
|
| 122 |
+
"""
|
| 123 |
+
if index < 0:
|
| 124 |
+
raise ValueError(f"MiniMax-H3 reference index must be nonnegative, got {index}")
|
| 125 |
+
if kind not in REFERENCE_KINDS:
|
| 126 |
+
raise ValueError(f"Unsupported MiniMax-H3 reference kind: {kind}")
|
| 127 |
+
return f"ref_{index:03d}_{kind}"
|
| 128 |
+
|
| 129 |
+
|
| 130 |
+
@dataclass(frozen=True)
|
| 131 |
+
class H3ConditionRole:
|
| 132 |
+
"""A parsed condition role name (see the vocabulary above).
|
| 133 |
+
|
| 134 |
+
``index`` is the one-frame slot or reference number (None for the ``first``/``last``
|
| 135 |
+
anchors); ``reference_kind`` is the reference row type (None outside the reference family).
|
| 136 |
+
"""
|
| 137 |
+
|
| 138 |
+
name: str
|
| 139 |
+
family: H3ConditionFamily
|
| 140 |
+
index: int | None = None
|
| 141 |
+
reference_kind: H3ReferenceKind | None = None
|
| 142 |
+
|
| 143 |
+
@property
|
| 144 |
+
def is_audio(self) -> bool:
|
| 145 |
+
return self.reference_kind == "audio"
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
def parse_condition_role(role: str) -> H3ConditionRole:
|
| 149 |
+
"""Parses a condition role name; raises ValueError for anything outside the vocabulary."""
|
| 150 |
+
if role in FL_CONDITION_ROLES:
|
| 151 |
+
return H3ConditionRole(role, "fl")
|
| 152 |
+
match = _ONE_FRAME_CONDITION_ROLE.fullmatch(role)
|
| 153 |
+
if match is not None:
|
| 154 |
+
return H3ConditionRole(role, "one_frame", index=int(match.group(1)))
|
| 155 |
+
match = _REFERENCE_CONDITION_ROLE.fullmatch(role)
|
| 156 |
+
if match is not None:
|
| 157 |
+
return H3ConditionRole(role, "reference", index=int(match.group(1)), reference_kind=match.group(2))
|
| 158 |
+
raise ValueError(f"Unsupported MiniMax-H3 condition role: {role}")
|
| 159 |
+
|
| 160 |
+
|
| 161 |
+
@dataclass(frozen=True)
|
| 162 |
+
class H3TimeOverrides:
|
| 163 |
+
"""Explicit RoPE times for a one-frame layout, in rotary units (1 unit = 1/40 s).
|
| 164 |
+
|
| 165 |
+
Times are relative to the target-block cursor (text end for fl2va, after the
|
| 166 |
+
reference blocks for ref2va); only relative placement carries meaning. A 24 fps
|
| 167 |
+
pixel-frame index maps to FRAME_RESCALE * index units.
|
| 168 |
+
"""
|
| 169 |
+
|
| 170 |
+
condition_times: tuple[float, ...]
|
| 171 |
+
target_time: float
|
| 172 |
+
|
| 173 |
+
def __post_init__(self) -> None:
|
| 174 |
+
for label, value in (
|
| 175 |
+
*((f"condition time {i}", t) for i, t in enumerate(self.condition_times)),
|
| 176 |
+
("target time", self.target_time),
|
| 177 |
+
):
|
| 178 |
+
if not math.isfinite(value) or value < 0.0:
|
| 179 |
+
raise ValueError(f"MiniMax-H3 {label} must be finite and nonnegative, got {value}")
|
| 180 |
+
|
| 181 |
+
|
| 182 |
+
@dataclass(frozen=True)
|
| 183 |
+
class H3RowSegment:
|
| 184 |
+
role: str
|
| 185 |
+
kind: H3SegmentKind
|
| 186 |
+
start: int
|
| 187 |
+
stop: int
|
| 188 |
+
|
| 189 |
+
def __post_init__(self) -> None:
|
| 190 |
+
if self.start < 0 or self.stop <= self.start:
|
| 191 |
+
raise ValueError(f"Invalid MiniMax-H3 row segment {self.role}: [{self.start}, {self.stop})")
|
| 192 |
+
|
| 193 |
+
@property
|
| 194 |
+
def row_count(self) -> int:
|
| 195 |
+
return self.stop - self.start
|
| 196 |
+
|
| 197 |
+
@property
|
| 198 |
+
def row_slice(self) -> slice:
|
| 199 |
+
return slice(self.start, self.stop)
|
| 200 |
+
|
| 201 |
+
|
| 202 |
+
@dataclass(frozen=True)
|
| 203 |
+
class H3PackedLayout:
|
| 204 |
+
task: H3Task
|
| 205 |
+
text_length: int
|
| 206 |
+
target_video: H3VideoGeometry
|
| 207 |
+
target_audio_frames: int
|
| 208 |
+
visual_conditions: tuple[H3VideoGeometry, ...]
|
| 209 |
+
references: tuple[H3ReferenceGeometry, ...]
|
| 210 |
+
segments: tuple[H3RowSegment, ...]
|
| 211 |
+
row_count: int
|
| 212 |
+
# part of the frozen layout value so the transformer's layout-keyed rotary cache
|
| 213 |
+
# cannot serve a grid built for different times
|
| 214 |
+
time_overrides: H3TimeOverrides | None = None
|
| 215 |
+
# experimental target-timeline stretch: generated pixel frames sample the timeline at
|
| 216 |
+
# output_fps instead of the native 24 fps, so each frame spans 24/output_fps native
|
| 217 |
+
# rotary steps and the clip covers frame_count/output_fps real seconds. References and
|
| 218 |
+
# conditions keep their native 24 fps spans; the target audio frame count must cover
|
| 219 |
+
# the stretched real duration. Frozen here so the rotary cache keys on it.
|
| 220 |
+
output_fps: int = TARGET_FPS
|
| 221 |
+
# with a stretch, rotate this many leading (highest-frequency) temporal RoPE bands
|
| 222 |
+
# by the UNSTRETCHED grid instead. Those bands have periods at or below the latent
|
| 223 |
+
# token spacing, so they cannot encode time between tokens -- they carry a per-token
|
| 224 |
+
# lattice phase that training bakes in against the native (1,4,4,4,4)-span grid;
|
| 225 |
+
# stretching scrambles it with the 17-pixel-frame group period (fading/stripes).
|
| 226 |
+
# ~3 is the sub-lattice band count for the released 16-band spectrum.
|
| 227 |
+
temporal_fine_bands: int = 0
|
| 228 |
+
|
| 229 |
+
def segment(self, role: str) -> H3RowSegment:
|
| 230 |
+
matches = [segment for segment in self.segments if segment.role == role]
|
| 231 |
+
if len(matches) != 1:
|
| 232 |
+
raise KeyError(f"MiniMax-H3 layout expected exactly one {role!r} segment, found {len(matches)}")
|
| 233 |
+
return matches[0]
|
| 234 |
+
|
| 235 |
+
@property
|
| 236 |
+
def temporal_stretch(self) -> float:
|
| 237 |
+
return TARGET_FPS / self.output_fps
|
| 238 |
+
|
| 239 |
+
@property
|
| 240 |
+
def target_video_segment(self) -> H3RowSegment:
|
| 241 |
+
return self.segment("target_video")
|
| 242 |
+
|
| 243 |
+
@property
|
| 244 |
+
def target_audio_segment(self) -> H3RowSegment:
|
| 245 |
+
return self.segment("target_audio")
|
| 246 |
+
|
| 247 |
+
|
| 248 |
+
@dataclass(frozen=True)
|
| 249 |
+
class H3TimestepRows:
|
| 250 |
+
unique_timesteps: torch.Tensor
|
| 251 |
+
row_timesteps: torch.Tensor
|
| 252 |
+
row_timestep_indices: torch.Tensor
|
| 253 |
+
token_tags: torch.Tensor
|
| 254 |
+
block_adaln_indices: torch.Tensor
|
| 255 |
+
block_segments: tuple[tuple[int, int, int], ...]
|
| 256 |
+
video_timestep_index: int
|
| 257 |
+
audio_timestep_index: int
|
| 258 |
+
|
| 259 |
+
|
| 260 |
+
def _coerce_video_geometry(value: H3VideoGeometry | Sequence[int], label: str) -> H3VideoGeometry:
|
| 261 |
+
if isinstance(value, H3VideoGeometry):
|
| 262 |
+
return value
|
| 263 |
+
if len(value) != 3:
|
| 264 |
+
raise ValueError(f"MiniMax-H3 {label} geometry must be (frames,height,width), got {tuple(value)}")
|
| 265 |
+
return H3VideoGeometry(*(int(part) for part in value))
|
| 266 |
+
|
| 267 |
+
|
| 268 |
+
def _validate_video_latent_frames(frames: int, label: str) -> None:
|
| 269 |
+
if frames < 2 or (frames - 2) % 5:
|
| 270 |
+
raise ValueError(f"MiniMax-H3 {label} latent frames must be 5*n+2, got {frames}")
|
| 271 |
+
|
| 272 |
+
|
| 273 |
+
def _expected_audio_frames(video_latent_frames: int, output_fps: int = TARGET_FPS) -> int:
|
| 274 |
+
_validate_video_latent_frames(video_latent_frames, "target video")
|
| 275 |
+
n = (video_latent_frames - 2) // 5
|
| 276 |
+
pixel_frames = 17 * n + 5
|
| 277 |
+
return audio_latent_frames(pixel_frames, output_fps=output_fps)
|
| 278 |
+
|
| 279 |
+
|
| 280 |
+
def pack_video_rows(latents: torch.Tensor) -> torch.Tensor:
|
| 281 |
+
"""Patch H3 video latents while preserving the leading batch axis."""
|
| 282 |
+
unbatched = latents.ndim == 4
|
| 283 |
+
if unbatched:
|
| 284 |
+
latents = latents.unsqueeze(0)
|
| 285 |
+
if latents.ndim != 5:
|
| 286 |
+
raise ValueError(f"MiniMax-H3 video latents must be [B,24,F,H,W], got {tuple(latents.shape)}")
|
| 287 |
+
batch, channels, frames, height, width = latents.shape
|
| 288 |
+
if channels != VIDEO_CHANNELS:
|
| 289 |
+
raise ValueError(f"MiniMax-H3 video latents need {VIDEO_CHANNELS} channels, got {channels}")
|
| 290 |
+
patch_t, patch_h, patch_w = VIDEO_PATCH_SIZE
|
| 291 |
+
if frames % patch_t or height % patch_h or width % patch_w:
|
| 292 |
+
raise ValueError(f"MiniMax-H3 video latent shape {frames}x{height}x{width} is not divisible by patch {VIDEO_PATCH_SIZE}")
|
| 293 |
+
rows = latents.reshape(
|
| 294 |
+
batch,
|
| 295 |
+
channels,
|
| 296 |
+
frames // patch_t,
|
| 297 |
+
patch_t,
|
| 298 |
+
height // patch_h,
|
| 299 |
+
patch_h,
|
| 300 |
+
width // patch_w,
|
| 301 |
+
patch_w,
|
| 302 |
+
)
|
| 303 |
+
rows = rows.permute(0, 2, 4, 6, 1, 3, 5, 7).reshape(
|
| 304 |
+
batch,
|
| 305 |
+
(frames // patch_t) * (height // patch_h) * (width // patch_w),
|
| 306 |
+
channels * patch_t * patch_h * patch_w,
|
| 307 |
+
)
|
| 308 |
+
return rows[0] if unbatched else rows
|
| 309 |
+
|
| 310 |
+
|
| 311 |
+
def pack_audio_rows(latents: torch.Tensor) -> torch.Tensor:
|
| 312 |
+
"""Pack stereo H3 audio in channel-major order."""
|
| 313 |
+
unbatched = latents.ndim == 3
|
| 314 |
+
if unbatched:
|
| 315 |
+
latents = latents.unsqueeze(0)
|
| 316 |
+
if latents.ndim != 4 or latents.shape[1] != AUDIO_CHANNELS or latents.shape[2] != STEREO_CHANNELS:
|
| 317 |
+
raise ValueError(f"MiniMax-H3 audio latents must be [B,32,2,A], got {tuple(latents.shape)}")
|
| 318 |
+
batch, channels, stereo, frames = latents.shape
|
| 319 |
+
rows = latents.permute(0, 2, 3, 1).reshape(batch, stereo * frames, channels)
|
| 320 |
+
return rows[0] if unbatched else rows
|
| 321 |
+
|
| 322 |
+
|
| 323 |
+
ONE_FRAME_VIDEO_LATENT_FRAMES = 1
|
| 324 |
+
ONE_FRAME_AUDIO_LATENT_FRAMES = 2 # (10 * 1 pixel frame + 3) // 6
|
| 325 |
+
|
| 326 |
+
|
| 327 |
+
def build_h3_layout(
|
| 328 |
+
*,
|
| 329 |
+
task: H3Task,
|
| 330 |
+
text_length: int,
|
| 331 |
+
target_video: H3VideoGeometry | Sequence[int],
|
| 332 |
+
target_audio_frames: int,
|
| 333 |
+
visual_conditions: Sequence[H3VideoGeometry | Sequence[int]] = (),
|
| 334 |
+
references: Sequence[H3ReferenceGeometry] = (),
|
| 335 |
+
one_frame: bool = False,
|
| 336 |
+
condition_roles: Sequence[str] | None = None,
|
| 337 |
+
time_overrides: H3TimeOverrides | None = None,
|
| 338 |
+
output_fps: int = TARGET_FPS,
|
| 339 |
+
temporal_fine_bands: int = 0,
|
| 340 |
+
) -> H3PackedLayout:
|
| 341 |
+
if task not in {"t2va", "fl2va", "ref2va"}:
|
| 342 |
+
raise ValueError(f"Unsupported MiniMax-H3 task: {task}")
|
| 343 |
+
if text_length <= 0:
|
| 344 |
+
raise ValueError(f"MiniMax-H3 text length must be positive, got {text_length}")
|
| 345 |
+
if isinstance(output_fps, bool) or not isinstance(output_fps, int) or output_fps <= 0:
|
| 346 |
+
raise ValueError(f"MiniMax-H3 output fps must be a positive integer, got {output_fps!r}")
|
| 347 |
+
if one_frame and output_fps != TARGET_FPS:
|
| 348 |
+
raise ValueError("MiniMax-H3 one-frame layouts do not support temporal stretch")
|
| 349 |
+
temporal_fine_bands = int(temporal_fine_bands)
|
| 350 |
+
if temporal_fine_bands < 0:
|
| 351 |
+
raise ValueError(f"MiniMax-H3 temporal fine bands must be nonnegative, got {temporal_fine_bands}")
|
| 352 |
+
if temporal_fine_bands and output_fps == TARGET_FPS:
|
| 353 |
+
raise ValueError("MiniMax-H3 temporal fine bands require an active temporal stretch")
|
| 354 |
+
target_video = _coerce_video_geometry(target_video, "target video")
|
| 355 |
+
if one_frame:
|
| 356 |
+
if target_video.frames != ONE_FRAME_VIDEO_LATENT_FRAMES:
|
| 357 |
+
raise ValueError(f"MiniMax-H3 one-frame layout requires a single target latent frame, got {target_video.frames}")
|
| 358 |
+
if target_audio_frames != ONE_FRAME_AUDIO_LATENT_FRAMES:
|
| 359 |
+
raise ValueError(
|
| 360 |
+
f"MiniMax-H3 one-frame layout requires {ONE_FRAME_AUDIO_LATENT_FRAMES} target audio frames, "
|
| 361 |
+
f"got {target_audio_frames}"
|
| 362 |
+
)
|
| 363 |
+
else:
|
| 364 |
+
if time_overrides is not None:
|
| 365 |
+
raise ValueError("MiniMax-H3 time overrides require a one-frame layout")
|
| 366 |
+
_validate_video_latent_frames(target_video.frames, "target video")
|
| 367 |
+
expected_audio_frames = _expected_audio_frames(target_video.frames, output_fps)
|
| 368 |
+
if target_audio_frames != expected_audio_frames:
|
| 369 |
+
raise ValueError(
|
| 370 |
+
f"MiniMax-H3 target audio has {target_audio_frames} frames, expected {expected_audio_frames} "
|
| 371 |
+
f"for {target_video.frames} video latent frames at {output_fps} fps"
|
| 372 |
+
)
|
| 373 |
+
|
| 374 |
+
visual_conditions = tuple(
|
| 375 |
+
_coerce_video_geometry(condition, f"visual condition {index}") for index, condition in enumerate(visual_conditions)
|
| 376 |
+
)
|
| 377 |
+
references = tuple(references)
|
| 378 |
+
if task == "t2va" and (visual_conditions or references):
|
| 379 |
+
raise ValueError("MiniMax-H3 T2VA layout does not accept condition rows")
|
| 380 |
+
if task == "fl2va":
|
| 381 |
+
if references:
|
| 382 |
+
raise ValueError("MiniMax-H3 FL2VA layout requires exactly first and last visual conditions")
|
| 383 |
+
if one_frame:
|
| 384 |
+
# ordered cond_{i} slots, any count, each placed by its own time override
|
| 385 |
+
if not visual_conditions:
|
| 386 |
+
raise ValueError("MiniMax-H3 one-frame FL2VA layout requires at least one visual condition")
|
| 387 |
+
if time_overrides is None or len(time_overrides.condition_times) != len(visual_conditions):
|
| 388 |
+
raise ValueError("MiniMax-H3 one-frame FL2VA layout requires one condition time override per condition")
|
| 389 |
+
roles = one_frame_condition_roles(len(visual_conditions))
|
| 390 |
+
if condition_roles is not None and tuple(condition_roles) != roles:
|
| 391 |
+
raise ValueError(
|
| 392 |
+
f"MiniMax-H3 one-frame FL2VA condition roles are the ordered {roles}, got {tuple(condition_roles)}"
|
| 393 |
+
)
|
| 394 |
+
else:
|
| 395 |
+
if not 1 <= len(visual_conditions) <= 2:
|
| 396 |
+
raise ValueError("MiniMax-H3 FL2VA layout requires one or two visual conditions (first and/or last)")
|
| 397 |
+
roles = _fl_condition_roles(condition_roles, len(visual_conditions))
|
| 398 |
+
for role, condition in zip(roles, visual_conditions):
|
| 399 |
+
if condition != H3VideoGeometry(1, target_video.height, target_video.width):
|
| 400 |
+
raise ValueError(f"MiniMax-H3 FL2VA {role} condition must be one target-sized latent frame, got {condition}")
|
| 401 |
+
else:
|
| 402 |
+
if condition_roles is not None:
|
| 403 |
+
raise ValueError("MiniMax-H3 condition roles apply only to FL2VA layouts")
|
| 404 |
+
if task == "ref2va":
|
| 405 |
+
if visual_conditions or not references:
|
| 406 |
+
raise ValueError("MiniMax-H3 Ref2VA layout requires ordered references and no FL2VA conditions")
|
| 407 |
+
if not any(reference.kind in {"image", "video"} for reference in references):
|
| 408 |
+
raise ValueError("MiniMax-H3 Ref2VA layout requires at least one visual reference")
|
| 409 |
+
if time_overrides is not None and task != "fl2va" and time_overrides.condition_times:
|
| 410 |
+
raise ValueError(f"MiniMax-H3 {task} time overrides cannot carry condition times")
|
| 411 |
+
|
| 412 |
+
segments = []
|
| 413 |
+
row = 0
|
| 414 |
+
|
| 415 |
+
def append(role: str, kind: H3SegmentKind, row_count: int) -> None:
|
| 416 |
+
nonlocal row
|
| 417 |
+
segments.append(H3RowSegment(role, kind, row, row + row_count))
|
| 418 |
+
row += row_count
|
| 419 |
+
|
| 420 |
+
append("text", "text", text_length)
|
| 421 |
+
if task == "fl2va":
|
| 422 |
+
for role, condition in zip(roles, visual_conditions):
|
| 423 |
+
append(role, "visual_condition", condition.row_count)
|
| 424 |
+
elif task == "ref2va":
|
| 425 |
+
for index, reference in enumerate(references):
|
| 426 |
+
if reference.kind == "image":
|
| 427 |
+
append(reference_condition_role(index, "image"), "visual_condition", reference.video.row_count)
|
| 428 |
+
elif reference.kind == "audio":
|
| 429 |
+
append(reference_condition_role(index, "audio"), "audio_condition", reference.audio_frames * STEREO_CHANNELS)
|
| 430 |
+
else:
|
| 431 |
+
if reference.audio_frames:
|
| 432 |
+
append(reference_condition_role(index, "audio"), "audio_condition", reference.audio_frames * STEREO_CHANNELS)
|
| 433 |
+
append(reference_condition_role(index, "video"), "visual_condition", reference.video.row_count)
|
| 434 |
+
append("target_audio", "target_audio", target_audio_frames * STEREO_CHANNELS)
|
| 435 |
+
append("target_video", "target_video", target_video.row_count)
|
| 436 |
+
|
| 437 |
+
return H3PackedLayout(
|
| 438 |
+
task=task,
|
| 439 |
+
text_length=text_length,
|
| 440 |
+
target_video=target_video,
|
| 441 |
+
target_audio_frames=target_audio_frames,
|
| 442 |
+
visual_conditions=visual_conditions,
|
| 443 |
+
references=references,
|
| 444 |
+
segments=tuple(segments),
|
| 445 |
+
row_count=row,
|
| 446 |
+
time_overrides=time_overrides,
|
| 447 |
+
output_fps=output_fps,
|
| 448 |
+
temporal_fine_bands=temporal_fine_bands,
|
| 449 |
+
)
|
| 450 |
+
|
| 451 |
+
|
| 452 |
+
def _fl_condition_roles(condition_roles: Sequence[str] | None, condition_count: int) -> tuple[str, ...]:
|
| 453 |
+
if condition_roles is None:
|
| 454 |
+
if condition_count != 2:
|
| 455 |
+
raise ValueError("MiniMax-H3 FL2VA layout with a single condition requires explicit condition roles")
|
| 456 |
+
return FL_CONDITION_ROLES
|
| 457 |
+
roles = tuple(condition_roles)
|
| 458 |
+
if roles not in {("first",), ("last",), FL_CONDITION_ROLES}:
|
| 459 |
+
raise ValueError(f"MiniMax-H3 FL2VA condition roles must be first, last, or first+last in order, got {roles}")
|
| 460 |
+
if len(roles) != condition_count:
|
| 461 |
+
raise ValueError(f"MiniMax-H3 FL2VA layout has {condition_count} conditions for {len(roles)} roles")
|
| 462 |
+
return roles
|
| 463 |
+
|
| 464 |
+
|
| 465 |
+
def _axis_from_sqrt_area(dimension: int, sqrt_area: float) -> torch.Tensor:
|
| 466 |
+
ratio = dimension / sqrt_area
|
| 467 |
+
count = dimension // 2
|
| 468 |
+
return (torch.arange(count, dtype=torch.float64) * (ratio / count) + (1.0 - ratio) / 2.0) * 32.0
|
| 469 |
+
|
| 470 |
+
|
| 471 |
+
def _frame_grid(video: H3VideoGeometry) -> tuple[torch.Tensor, torch.Tensor]:
|
| 472 |
+
sqrt_area = math.sqrt(video.height * video.width)
|
| 473 |
+
height_axis = _axis_from_sqrt_area(video.height, sqrt_area)
|
| 474 |
+
width_axis = _axis_from_sqrt_area(video.width, sqrt_area)
|
| 475 |
+
height, width = torch.meshgrid(height_axis, width_axis, indexing="ij")
|
| 476 |
+
return torch.stack((height.reshape(-1), width.reshape(-1)), dim=-1), width_axis
|
| 477 |
+
|
| 478 |
+
|
| 479 |
+
def _video_t_spans(frames: int, temporal_stretch: float = 1.0) -> list[float]:
|
| 480 |
+
return [temporal_stretch * FRAME_RESCALE * FRAME_PER_TOKEN[index % len(FRAME_PER_TOKEN)] for index in range(frames)]
|
| 481 |
+
|
| 482 |
+
|
| 483 |
+
def _video_grid(video: H3VideoGeometry, cursor: float, temporal_stretch: float = 1.0) -> torch.Tensor:
|
| 484 |
+
spans = torch.tensor(_video_t_spans(video.frames, temporal_stretch), dtype=torch.float64)
|
| 485 |
+
times = cursor + torch.cat((torch.zeros(1, dtype=torch.float64), spans[:-1].cumsum(0)))
|
| 486 |
+
frame, _ = _frame_grid(video)
|
| 487 |
+
grid = torch.empty(video.frames, frame.shape[0], 3, dtype=torch.float64)
|
| 488 |
+
grid[..., 0] = times[:, None]
|
| 489 |
+
grid[..., 1:] = frame[None]
|
| 490 |
+
return grid.reshape(-1, 3)
|
| 491 |
+
|
| 492 |
+
|
| 493 |
+
def _audio_grid(cursor: float, frames: int, width_low: float, width_high: float) -> torch.Tensor:
|
| 494 |
+
grid = torch.zeros(frames * STEREO_CHANNELS, 3, dtype=torch.float64)
|
| 495 |
+
grid[:, 0] = (cursor + torch.arange(frames, dtype=torch.float64)).repeat(STEREO_CHANNELS)
|
| 496 |
+
grid[:frames, 2] = width_low
|
| 497 |
+
grid[frames:, 2] = width_high
|
| 498 |
+
return grid
|
| 499 |
+
|
| 500 |
+
|
| 501 |
+
def build_position_grid(layout: H3PackedLayout, *, device: torch.device | str | None = None) -> torch.Tensor:
|
| 502 |
+
positions = torch.empty(layout.row_count, 3, dtype=torch.float64)
|
| 503 |
+
text = layout.segment("text")
|
| 504 |
+
positions[text.row_slice] = 0
|
| 505 |
+
positions[text.row_slice, 0] = torch.arange(layout.text_length, dtype=torch.float64)
|
| 506 |
+
|
| 507 |
+
target_frame, target_width = _frame_grid(layout.target_video)
|
| 508 |
+
target_width_endpoints = float(target_width[0]), float(target_width[-1])
|
| 509 |
+
cursor = float(layout.text_length)
|
| 510 |
+
|
| 511 |
+
if layout.task == "fl2va":
|
| 512 |
+
condition_segments = tuple(segment for segment in layout.segments if segment.kind == "visual_condition")
|
| 513 |
+
if layout.time_overrides is not None:
|
| 514 |
+
condition_times = [cursor + time for time in layout.time_overrides.condition_times]
|
| 515 |
+
else:
|
| 516 |
+
# the last-frame anchor sits on the final pixel frame of the target timeline,
|
| 517 |
+
# so both the total span and the one-frame back-off scale with the stretch
|
| 518 |
+
last_time = cursor + sum(_video_t_spans(layout.target_video.frames, layout.temporal_stretch))
|
| 519 |
+
last_time -= FRAME_RESCALE * layout.temporal_stretch
|
| 520 |
+
condition_times = [cursor if segment.role == "first" else last_time for segment in condition_segments]
|
| 521 |
+
for segment, time in zip(condition_segments, condition_times):
|
| 522 |
+
positions[segment.row_slice, 0] = time
|
| 523 |
+
positions[segment.row_slice, 1:] = target_frame
|
| 524 |
+
elif layout.task == "ref2va":
|
| 525 |
+
for index, reference in enumerate(layout.references):
|
| 526 |
+
if reference.kind == "image":
|
| 527 |
+
segment = layout.segment(reference_condition_role(index, "image"))
|
| 528 |
+
frame, _ = _frame_grid(reference.video)
|
| 529 |
+
positions[segment.row_slice, 0] = cursor
|
| 530 |
+
positions[segment.row_slice, 1:] = frame
|
| 531 |
+
cursor += 1.0
|
| 532 |
+
elif reference.kind == "audio":
|
| 533 |
+
segment = layout.segment(reference_condition_role(index, "audio"))
|
| 534 |
+
positions[segment.row_slice] = _audio_grid(cursor, reference.audio_frames, *target_width_endpoints)
|
| 535 |
+
cursor += float(reference.audio_frames)
|
| 536 |
+
else:
|
| 537 |
+
frame, width = _frame_grid(reference.video)
|
| 538 |
+
if reference.audio_frames:
|
| 539 |
+
audio = layout.segment(reference_condition_role(index, "audio"))
|
| 540 |
+
positions[audio.row_slice] = _audio_grid(
|
| 541 |
+
cursor,
|
| 542 |
+
reference.audio_frames,
|
| 543 |
+
float(width[0]),
|
| 544 |
+
float(width[-1]),
|
| 545 |
+
)
|
| 546 |
+
video = layout.segment(reference_condition_role(index, "video"))
|
| 547 |
+
positions[video.row_slice] = _video_grid(reference.video, cursor)
|
| 548 |
+
cursor += max(float(reference.audio_frames), sum(_video_t_spans(reference.video.frames)))
|
| 549 |
+
|
| 550 |
+
if layout.time_overrides is not None:
|
| 551 |
+
cursor += layout.time_overrides.target_time
|
| 552 |
+
target_audio = layout.target_audio_segment
|
| 553 |
+
positions[target_audio.row_slice] = _audio_grid(
|
| 554 |
+
cursor,
|
| 555 |
+
layout.target_audio_frames,
|
| 556 |
+
*target_width_endpoints,
|
| 557 |
+
)
|
| 558 |
+
target_video = layout.target_video_segment
|
| 559 |
+
positions[target_video.row_slice] = _video_grid(layout.target_video, cursor, layout.temporal_stretch)
|
| 560 |
+
positions = positions.unsqueeze(0)
|
| 561 |
+
return positions.to(device=device) if device is not None else positions
|
| 562 |
+
|
| 563 |
+
|
| 564 |
+
def _single_model_time(value: float | torch.Tensor, label: str) -> float:
|
| 565 |
+
tensor = torch.as_tensor(value).detach().reshape(-1)
|
| 566 |
+
if tensor.numel() != 1:
|
| 567 |
+
raise ValueError(f"MiniMax-H3 R1 {label} must contain exactly one value")
|
| 568 |
+
scalar = float(tensor[0].item())
|
| 569 |
+
if not math.isfinite(scalar) or not 0.0 <= scalar <= 1.0:
|
| 570 |
+
raise ValueError(f"MiniMax-H3 {label} must be finite and in [0,1], got {scalar}")
|
| 571 |
+
return scalar
|
| 572 |
+
|
| 573 |
+
|
| 574 |
+
def validate_clean_coefficient(value: float, label: str) -> float:
|
| 575 |
+
"""The condition-augmentation clean coefficient (clean*x + (1-clean)*eps) must lie in [0,1];
|
| 576 |
+
the timestep rows, the samplers and the CLIs all validate through here."""
|
| 577 |
+
value = float(value)
|
| 578 |
+
if not math.isfinite(value) or not 0.0 <= value <= 1.0:
|
| 579 |
+
raise ValueError(f"MiniMax-H3 {label} must be finite and in [0,1], got {value}")
|
| 580 |
+
return value
|
| 581 |
+
|
| 582 |
+
|
| 583 |
+
def _build_modulation_segments(indices: torch.Tensor) -> tuple[tuple[int, int, int], ...]:
|
| 584 |
+
change_points = torch.nonzero(indices[1:] != indices[:-1], as_tuple=False).flatten() + 1
|
| 585 |
+
boundaries = torch.cat((indices.new_tensor([0]), change_points, indices.new_tensor([indices.numel()])))
|
| 586 |
+
return tuple(
|
| 587 |
+
tuple(row)
|
| 588 |
+
for row in torch.stack(
|
| 589 |
+
(boundaries[:-1], boundaries[1:], indices[boundaries[:-1]]),
|
| 590 |
+
dim=1,
|
| 591 |
+
).tolist()
|
| 592 |
+
)
|
| 593 |
+
|
| 594 |
+
|
| 595 |
+
def build_timestep_rows(
|
| 596 |
+
layout: H3PackedLayout,
|
| 597 |
+
*,
|
| 598 |
+
text_token_tags: torch.Tensor,
|
| 599 |
+
model_t_video: float | torch.Tensor,
|
| 600 |
+
model_t_audio: float | torch.Tensor,
|
| 601 |
+
visual_condition_clean: float = 0.999,
|
| 602 |
+
audio_condition_clean: float = 1.0,
|
| 603 |
+
) -> H3TimestepRows:
|
| 604 |
+
video_time = _single_model_time(model_t_video, "video model time")
|
| 605 |
+
audio_time = _single_model_time(model_t_audio, "audio model time")
|
| 606 |
+
visual_clean = validate_clean_coefficient(visual_condition_clean, "visual condition clean coefficient")
|
| 607 |
+
audio_clean = validate_clean_coefficient(audio_condition_clean, "audio condition clean coefficient")
|
| 608 |
+
|
| 609 |
+
text_token_tags = torch.as_tensor(text_token_tags)
|
| 610 |
+
if text_token_tags.dtype != torch.int64 or text_token_tags.shape != (1, layout.text_length):
|
| 611 |
+
raise ValueError(f"MiniMax-H3 text token tags must be int64 [1,{layout.text_length}]")
|
| 612 |
+
if not torch.all((text_token_tags == 0) | (text_token_tags == 1)):
|
| 613 |
+
raise ValueError("MiniMax-H3 text token tags may contain only 0 and 1")
|
| 614 |
+
text_token_tags = text_token_tags.detach().cpu()
|
| 615 |
+
|
| 616 |
+
segment_times = {}
|
| 617 |
+
for segment in layout.segments:
|
| 618 |
+
if segment.kind in {"text", "target_video"}:
|
| 619 |
+
segment_times[segment.role] = video_time
|
| 620 |
+
elif segment.kind == "target_audio":
|
| 621 |
+
segment_times[segment.role] = audio_time
|
| 622 |
+
elif segment.kind == "visual_condition":
|
| 623 |
+
segment_times[segment.role] = max(video_time, visual_clean)
|
| 624 |
+
else:
|
| 625 |
+
segment_times[segment.role] = max(audio_time, audio_clean)
|
| 626 |
+
|
| 627 |
+
unique_values = sorted(set(segment_times.values()))
|
| 628 |
+
timestep_index = {value: index for index, value in enumerate(unique_values)}
|
| 629 |
+
row_timesteps = torch.empty((1, layout.row_count), dtype=torch.float32)
|
| 630 |
+
row_timestep_indices = torch.empty((1, layout.row_count), dtype=torch.int64)
|
| 631 |
+
token_tags = torch.empty((1, layout.row_count), dtype=torch.int64)
|
| 632 |
+
for segment in layout.segments:
|
| 633 |
+
value = segment_times[segment.role]
|
| 634 |
+
row_timesteps[:, segment.row_slice] = value
|
| 635 |
+
row_timestep_indices[:, segment.row_slice] = timestep_index[value]
|
| 636 |
+
if segment.kind == "text":
|
| 637 |
+
token_tags[:, segment.row_slice] = text_token_tags
|
| 638 |
+
elif segment.kind in {"visual_condition", "target_video"}:
|
| 639 |
+
token_tags[:, segment.row_slice] = 0
|
| 640 |
+
else:
|
| 641 |
+
token_tags[:, segment.row_slice] = 2
|
| 642 |
+
|
| 643 |
+
block_adaln_indices = 3 * row_timestep_indices + token_tags
|
| 644 |
+
block_segments = _build_modulation_segments(block_adaln_indices[0])
|
| 645 |
+
|
| 646 |
+
return H3TimestepRows(
|
| 647 |
+
unique_timesteps=torch.tensor(unique_values, dtype=torch.float32),
|
| 648 |
+
row_timesteps=row_timesteps,
|
| 649 |
+
row_timestep_indices=row_timestep_indices,
|
| 650 |
+
token_tags=token_tags,
|
| 651 |
+
block_adaln_indices=block_adaln_indices,
|
| 652 |
+
block_segments=block_segments,
|
| 653 |
+
video_timestep_index=timestep_index[video_time],
|
| 654 |
+
audio_timestep_index=timestep_index[audio_time],
|
| 655 |
+
)
|
| 656 |
+
|
| 657 |
+
|
| 658 |
+
def _unpack_video_rows(rows: torch.Tensor, video: H3VideoGeometry) -> torch.Tensor:
|
| 659 |
+
unbatched = rows.ndim == 2
|
| 660 |
+
if unbatched:
|
| 661 |
+
rows = rows.unsqueeze(0)
|
| 662 |
+
expected_shape = (video.row_count, VIDEO_CHANNELS * math.prod(VIDEO_PATCH_SIZE))
|
| 663 |
+
if rows.ndim != 3 or tuple(rows.shape[1:]) != expected_shape:
|
| 664 |
+
raise ValueError(f"MiniMax-H3 target video rows must be [B,{expected_shape[0]},{expected_shape[1]}]")
|
| 665 |
+
batch = rows.shape[0]
|
| 666 |
+
patch_t, patch_h, patch_w = VIDEO_PATCH_SIZE
|
| 667 |
+
unpacked = rows.reshape(
|
| 668 |
+
batch,
|
| 669 |
+
video.frames // patch_t,
|
| 670 |
+
video.height // patch_h,
|
| 671 |
+
video.width // patch_w,
|
| 672 |
+
VIDEO_CHANNELS,
|
| 673 |
+
patch_t,
|
| 674 |
+
patch_h,
|
| 675 |
+
patch_w,
|
| 676 |
+
)
|
| 677 |
+
unpacked = unpacked.permute(0, 4, 1, 5, 2, 6, 3, 7).reshape(
|
| 678 |
+
batch,
|
| 679 |
+
VIDEO_CHANNELS,
|
| 680 |
+
video.frames,
|
| 681 |
+
video.height,
|
| 682 |
+
video.width,
|
| 683 |
+
)
|
| 684 |
+
return unpacked[0] if unbatched else unpacked
|
| 685 |
+
|
| 686 |
+
|
| 687 |
+
def _unpack_audio_rows(rows: torch.Tensor, audio_frames: int) -> torch.Tensor:
|
| 688 |
+
unbatched = rows.ndim == 2
|
| 689 |
+
if unbatched:
|
| 690 |
+
rows = rows.unsqueeze(0)
|
| 691 |
+
expected_shape = (audio_frames * STEREO_CHANNELS, AUDIO_CHANNELS)
|
| 692 |
+
if rows.ndim != 3 or tuple(rows.shape[1:]) != expected_shape:
|
| 693 |
+
raise ValueError(f"MiniMax-H3 target audio rows must be [B,{expected_shape[0]},{expected_shape[1]}]")
|
| 694 |
+
batch = rows.shape[0]
|
| 695 |
+
unpacked = rows.reshape(batch, STEREO_CHANNELS, audio_frames, AUDIO_CHANNELS).permute(0, 3, 1, 2)
|
| 696 |
+
return unpacked[0] if unbatched else unpacked
|
| 697 |
+
|
| 698 |
+
|
| 699 |
+
def unpack_targets(
|
| 700 |
+
layout: H3PackedLayout,
|
| 701 |
+
target_video_rows: torch.Tensor,
|
| 702 |
+
target_audio_rows: torch.Tensor,
|
| 703 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 704 |
+
if target_video_rows.ndim != target_audio_rows.ndim or target_video_rows.ndim not in {2, 3}:
|
| 705 |
+
raise ValueError("MiniMax-H3 target video/audio rows must both be batched or both be unbatched")
|
| 706 |
+
if target_video_rows.ndim == 3 and target_video_rows.shape[0] != target_audio_rows.shape[0]:
|
| 707 |
+
raise ValueError("MiniMax-H3 target video/audio rows must have matching batch axes")
|
| 708 |
+
return (
|
| 709 |
+
_unpack_video_rows(target_video_rows, layout.target_video),
|
| 710 |
+
_unpack_audio_rows(target_audio_rows, layout.target_audio_frames),
|
| 711 |
+
)
|
third_party/musubi_tuner/minimax_h3/text_encoder.py
ADDED
|
@@ -0,0 +1,712 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2026 The MiniMax and HuggingFace Teams. All rights reserved.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
#
|
| 15 |
+
# Adapted for Musubi from Hugging Face Diffusers PR #14355 at commit
|
| 16 |
+
# abc5e9bf71fd38f53cd471bc3acaa84bc5ecbfdc
|
| 17 |
+
# (modular_pipelines/minimax_h3/encoders.py and packing_ref2va.py).
|
| 18 |
+
# ComfyUI is used only as an independent numerical reference.
|
| 19 |
+
|
| 20 |
+
from __future__ import annotations
|
| 21 |
+
|
| 22 |
+
from collections.abc import Mapping
|
| 23 |
+
from dataclasses import dataclass
|
| 24 |
+
import hashlib
|
| 25 |
+
import json
|
| 26 |
+
import logging
|
| 27 |
+
from pathlib import Path
|
| 28 |
+
from typing import Any
|
| 29 |
+
|
| 30 |
+
import numpy as np
|
| 31 |
+
import torch
|
| 32 |
+
import torch.nn as nn
|
| 33 |
+
|
| 34 |
+
from musubi_tuner.minimax_h3.checkpoint import resolve_safetensors_files
|
| 35 |
+
from musubi_tuner.minimax_h3.media import TEXT_VISUAL_FPS, H3Record, H3Task
|
| 36 |
+
from musubi_tuner.minimax_h3.packing import FL_CONDITION_ROLES, one_frame_condition_roles, parse_condition_role
|
| 37 |
+
|
| 38 |
+
logger = logging.getLogger(__name__)
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
TEXT_WIDTH = 5120
|
| 42 |
+
MAX_TEXT_ROWS = 32768
|
| 43 |
+
LAYER_50_HIDDEN_STATE_INDEX = 50
|
| 44 |
+
VISION_START_TOKEN_ID = 151652
|
| 45 |
+
VISION_END_TOKEN_ID = 151653
|
| 46 |
+
IMAGE_TOKEN_ID = 151655
|
| 47 |
+
VIDEO_TOKEN_ID = 151656
|
| 48 |
+
IMAGE_PLACEHOLDER = "<|vision_start|><|image_pad|><|vision_end|>"
|
| 49 |
+
VIDEO_PLACEHOLDER = "<|vision_start|><|video_pad|><|vision_end|>"
|
| 50 |
+
# the H3 encoder is Qwen3-VL-32B, but the released repository ships its own processor and config:
|
| 51 |
+
# the tokenizer adds <d>, </d>, <|cutoff|>, <|lyrics_start|>, <|lyrics_end|>, <|caption_start|> and
|
| 52 |
+
# <|caption_end|> as special tokens (ids 151669-151675), which Qwen/Qwen3-VL-32B-Instruct splits into
|
| 53 |
+
# ordinary tokens instead. The released prompt format writes dialogue and lyrics as <d>[Language] ...</d>,
|
| 54 |
+
# so the H3 files are required, not interchangeable with the upstream Qwen ones.
|
| 55 |
+
H3_REPO_ID = "MiniMaxAI/MiniMax-H3"
|
| 56 |
+
PROCESSOR_SUBFOLDER = "processor"
|
| 57 |
+
TEXT_ENCODER_CONFIG_SUBFOLDER = "text_encoder"
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
@dataclass(frozen=True)
|
| 61 |
+
class H3TextVisual:
|
| 62 |
+
frames: torch.Tensor
|
| 63 |
+
timestamps: tuple[float, ...] | None = None
|
| 64 |
+
|
| 65 |
+
def __post_init__(self) -> None:
|
| 66 |
+
if self.frames.ndim != 4 or self.frames.shape[-1] < 3 or self.frames.shape[0] == 0:
|
| 67 |
+
raise ValueError(f"MiniMax-H3 text visual must be [T,H,W,C], got {tuple(self.frames.shape)}")
|
| 68 |
+
if self.timestamps is not None and len(self.timestamps) != self.frames.shape[0]:
|
| 69 |
+
raise ValueError("MiniMax-H3 text visual timestamps must match its frame count")
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
@dataclass(frozen=True)
|
| 73 |
+
class H3Presentation:
|
| 74 |
+
text: str
|
| 75 |
+
images: tuple[torch.Tensor, ...] = ()
|
| 76 |
+
videos: tuple[torch.Tensor, ...] = ()
|
| 77 |
+
processor_text: str | None = None
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def _require_visual(visuals: Mapping[object, H3TextVisual], key: object, label: str) -> H3TextVisual:
|
| 81 |
+
try:
|
| 82 |
+
return visuals[key]
|
| 83 |
+
except KeyError as error:
|
| 84 |
+
raise ValueError(f"MiniMax-H3 presentation is missing {label} visual data") from error
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def _fl_visual_keys(visuals: Mapping[object, H3TextVisual]) -> list[str]:
|
| 88 |
+
"""The FL2VA visual keys present, in <Picture i> order: the released first/last anchors or
|
| 89 |
+
the contiguous one-frame cond_{i} slots (the same role vocabulary as the latent cache)."""
|
| 90 |
+
present_keys = [key for key in FL_CONDITION_ROLES if key in visuals]
|
| 91 |
+
cond_keys = sorted(key for key in visuals if isinstance(key, str) and _is_one_frame_condition_role(key))
|
| 92 |
+
if present_keys and cond_keys:
|
| 93 |
+
raise ValueError("MiniMax-H3 FL2VA presentation cannot mix first/last visuals with one-frame cond_ visuals")
|
| 94 |
+
if cond_keys:
|
| 95 |
+
expected = list(one_frame_condition_roles(len(cond_keys)))
|
| 96 |
+
if cond_keys != expected:
|
| 97 |
+
raise ValueError(f"MiniMax-H3 one-frame FL2VA visuals must be the contiguous {expected}, got {cond_keys}")
|
| 98 |
+
present_keys = cond_keys
|
| 99 |
+
if not present_keys:
|
| 100 |
+
raise ValueError(
|
| 101 |
+
"MiniMax-H3 FL2VA presentation requires the first/last visuals (video targets)"
|
| 102 |
+
" or the cond_{i} control visuals (one-frame targets)"
|
| 103 |
+
)
|
| 104 |
+
return present_keys
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
def _is_one_frame_condition_role(key: str) -> bool:
|
| 108 |
+
try:
|
| 109 |
+
return parse_condition_role(key).family == "one_frame"
|
| 110 |
+
except ValueError:
|
| 111 |
+
return False
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def build_presentation(
|
| 115 |
+
record: H3Record,
|
| 116 |
+
task: H3Task,
|
| 117 |
+
visuals: Mapping[object, H3TextVisual] | None = None,
|
| 118 |
+
) -> H3Presentation:
|
| 119 |
+
if task not in {"t2va", "fl2va", "ref2va"}:
|
| 120 |
+
raise ValueError(f"Unsupported MiniMax-H3 task: {task}")
|
| 121 |
+
visuals = visuals or {}
|
| 122 |
+
if task != "ref2va" and record.references:
|
| 123 |
+
raise ValueError(f"MiniMax-H3 task {task} does not accept references")
|
| 124 |
+
if task == "t2va":
|
| 125 |
+
return H3Presentation(text=record.caption, processor_text=record.caption)
|
| 126 |
+
|
| 127 |
+
parts = []
|
| 128 |
+
processor_parts = []
|
| 129 |
+
images = []
|
| 130 |
+
videos = []
|
| 131 |
+
if task == "fl2va":
|
| 132 |
+
# the released builder numbers <Picture i> over the pictures that are present,
|
| 133 |
+
# in packed (first, last) order: a lone last frame is still <Picture 1>, and the
|
| 134 |
+
# first/last distinction is carried only by the rotary anchor times. One-frame
|
| 135 |
+
# layouts use the ordered cond_{i} slots instead, numbered in slot order.
|
| 136 |
+
for index, key in enumerate(_fl_visual_keys(visuals), start=1):
|
| 137 |
+
visual = visuals[key]
|
| 138 |
+
if visual.frames.shape[0] != 1:
|
| 139 |
+
raise ValueError(f"MiniMax-H3 FL2VA {key} visual must contain exactly one frame")
|
| 140 |
+
part = f"<Picture {index}>: {IMAGE_PLACEHOLDER}"
|
| 141 |
+
parts.append(part)
|
| 142 |
+
processor_parts.append(part)
|
| 143 |
+
images.append(visual.frames[0])
|
| 144 |
+
parts.append(record.caption)
|
| 145 |
+
processor_parts.append(record.caption)
|
| 146 |
+
return H3Presentation(
|
| 147 |
+
text="".join(parts),
|
| 148 |
+
images=tuple(images),
|
| 149 |
+
processor_text="".join(processor_parts),
|
| 150 |
+
)
|
| 151 |
+
|
| 152 |
+
counters = {"image": 0, "audio": 0, "video": 0}
|
| 153 |
+
for reference in record.references:
|
| 154 |
+
if reference.type == "image":
|
| 155 |
+
counters["image"] += 1
|
| 156 |
+
visual = _require_visual(visuals, reference.path, f"reference image {reference.path}")
|
| 157 |
+
if visual.frames.shape[0] != 1:
|
| 158 |
+
raise ValueError(f"MiniMax-H3 reference image must contain one frame: {reference.path}")
|
| 159 |
+
part = f"<Picture {counters['image']}>: {IMAGE_PLACEHOLDER}"
|
| 160 |
+
parts.append(part)
|
| 161 |
+
processor_parts.append(part)
|
| 162 |
+
images.append(visual.frames[0])
|
| 163 |
+
continue
|
| 164 |
+
|
| 165 |
+
if reference.type == "audio":
|
| 166 |
+
counters["audio"] += 1
|
| 167 |
+
part = f"<Audio {counters['audio']}>: "
|
| 168 |
+
parts.append(part)
|
| 169 |
+
processor_parts.append(part)
|
| 170 |
+
continue
|
| 171 |
+
|
| 172 |
+
if reference.type != "video":
|
| 173 |
+
raise ValueError(f"Unsupported MiniMax-H3 reference type: {reference.type}")
|
| 174 |
+
if reference.audio is not None:
|
| 175 |
+
counters["audio"] += 1
|
| 176 |
+
part = f"<Audio {counters['audio']}>: "
|
| 177 |
+
parts.append(part)
|
| 178 |
+
processor_parts.append(part)
|
| 179 |
+
counters["video"] += 1
|
| 180 |
+
part = f"<Video {counters['video']}>: "
|
| 181 |
+
parts.append(part)
|
| 182 |
+
processor_parts.append(part)
|
| 183 |
+
processor_parts.append(VIDEO_PLACEHOLDER)
|
| 184 |
+
|
| 185 |
+
visual = _require_visual(visuals, reference.path, f"reference video {reference.path}")
|
| 186 |
+
frames = visual.frames
|
| 187 |
+
timestamps = (
|
| 188 |
+
list(visual.timestamps) if visual.timestamps is not None else [index / TEXT_VISUAL_FPS for index in range(len(frames))]
|
| 189 |
+
)
|
| 190 |
+
if len(frames) % 2:
|
| 191 |
+
frames = torch.cat((frames, frames[-1:]), dim=0)
|
| 192 |
+
timestamps.append(timestamps[-1])
|
| 193 |
+
for index in range(0, len(frames), 2):
|
| 194 |
+
block_timestamp = (timestamps[index] + timestamps[index + 1]) / 2.0
|
| 195 |
+
parts.append(f"<{block_timestamp:.1f} seconds>{VIDEO_PLACEHOLDER}")
|
| 196 |
+
videos.append(visual.frames)
|
| 197 |
+
|
| 198 |
+
parts.append(record.caption)
|
| 199 |
+
processor_parts.append(record.caption)
|
| 200 |
+
return H3Presentation(
|
| 201 |
+
text="".join(parts),
|
| 202 |
+
images=tuple(images),
|
| 203 |
+
videos=tuple(videos),
|
| 204 |
+
processor_text="".join(processor_parts),
|
| 205 |
+
)
|
| 206 |
+
|
| 207 |
+
|
| 208 |
+
def build_token_tags(processed: Mapping[str, Any]) -> torch.Tensor:
|
| 209 |
+
if "input_ids" not in processed:
|
| 210 |
+
raise ValueError("MiniMax-H3 processor output has no input_ids")
|
| 211 |
+
input_ids = torch.as_tensor(processed["input_ids"])
|
| 212 |
+
if input_ids.ndim == 2:
|
| 213 |
+
if input_ids.shape[0] != 1:
|
| 214 |
+
raise ValueError("MiniMax-H3 text caching processes one presentation at a time")
|
| 215 |
+
input_ids = input_ids[0]
|
| 216 |
+
if input_ids.ndim != 1:
|
| 217 |
+
raise ValueError(f"MiniMax-H3 input_ids must be [L] or [1,L], got {tuple(input_ids.shape)}")
|
| 218 |
+
|
| 219 |
+
tags = torch.ones(input_ids.shape[0], dtype=torch.int64, device=input_ids.device)
|
| 220 |
+
open_start = None
|
| 221 |
+
for index, token_id in enumerate(input_ids.tolist()):
|
| 222 |
+
if token_id == VISION_START_TOKEN_ID:
|
| 223 |
+
if open_start is not None:
|
| 224 |
+
raise ValueError("MiniMax-H3 processor produced nested vision-start tokens")
|
| 225 |
+
open_start = index
|
| 226 |
+
elif token_id == VISION_END_TOKEN_ID:
|
| 227 |
+
if open_start is None:
|
| 228 |
+
raise ValueError("MiniMax-H3 processor produced an unmatched vision-end token")
|
| 229 |
+
interior = input_ids[open_start + 1 : index]
|
| 230 |
+
if not torch.any((interior == IMAGE_TOKEN_ID) | (interior == VIDEO_TOKEN_ID)):
|
| 231 |
+
raise ValueError("MiniMax-H3 vision span contains no expanded image or video rows")
|
| 232 |
+
tags[open_start : index + 1] = 0
|
| 233 |
+
open_start = None
|
| 234 |
+
if open_start is not None:
|
| 235 |
+
raise ValueError("MiniMax-H3 processor produced an unmatched vision-start token")
|
| 236 |
+
return tags.cpu()
|
| 237 |
+
|
| 238 |
+
|
| 239 |
+
def _text_size_error(row_count: int, token_tags: torch.Tensor) -> ValueError:
|
| 240 |
+
vision_rows = int((token_tags == 0).sum().item())
|
| 241 |
+
text_rows = row_count - vision_rows
|
| 242 |
+
payload_mib = row_count * TEXT_WIDTH * 2 / (1024**2)
|
| 243 |
+
return ValueError(
|
| 244 |
+
f"MiniMax-H3 text presentation has {row_count} rows, exceeds {MAX_TEXT_ROWS}; "
|
| 245 |
+
f"vision_rows={vision_rows}, text_rows={text_rows}, estimated BF16 payload={payload_mib:.1f} MiB"
|
| 246 |
+
)
|
| 247 |
+
|
| 248 |
+
|
| 249 |
+
def validate_text_rows(hidden_states: torch.Tensor, token_tags: torch.Tensor) -> None:
|
| 250 |
+
if hidden_states.ndim != 2 or hidden_states.shape[1] != TEXT_WIDTH:
|
| 251 |
+
raise ValueError(f"Expected MiniMax-H3 hidden states [L,{TEXT_WIDTH}], got {tuple(hidden_states.shape)}")
|
| 252 |
+
if token_tags.dtype != torch.int64 or token_tags.shape != (hidden_states.shape[0],):
|
| 253 |
+
raise ValueError("MiniMax-H3 token tags must be int64 [L]")
|
| 254 |
+
if hidden_states.shape[0] > MAX_TEXT_ROWS:
|
| 255 |
+
raise _text_size_error(hidden_states.shape[0], token_tags)
|
| 256 |
+
if not torch.all((token_tags == 0) | (token_tags == 1)):
|
| 257 |
+
raise ValueError("MiniMax-H3 token tags may contain only 0 and 1")
|
| 258 |
+
|
| 259 |
+
|
| 260 |
+
def _language_layers(model) -> list:
|
| 261 |
+
language_model = getattr(model, "language_model", None)
|
| 262 |
+
layers = getattr(language_model, "layers", None)
|
| 263 |
+
if layers is None:
|
| 264 |
+
raise ValueError("MiniMax-H3 Qwen3-VL model has no language_model.layers")
|
| 265 |
+
return layers
|
| 266 |
+
|
| 267 |
+
|
| 268 |
+
def normalize_h3_text_encoder_key(key: str) -> str:
|
| 269 |
+
if key.startswith("model."):
|
| 270 |
+
return "language_model." + key.removeprefix("model.")
|
| 271 |
+
return key
|
| 272 |
+
|
| 273 |
+
|
| 274 |
+
def load_h3_processor():
|
| 275 |
+
# the concrete class, not AutoProcessor: the auto class resolves the processor type from the
|
| 276 |
+
# repository root, where the H3 release keeps a diffusers model_index.json instead of a config
|
| 277 |
+
from transformers import Qwen3VLProcessor
|
| 278 |
+
|
| 279 |
+
return Qwen3VLProcessor.from_pretrained(H3_REPO_ID, subfolder=PROCESSOR_SUBFOLDER)
|
| 280 |
+
|
| 281 |
+
|
| 282 |
+
def load_h3_text_encoder(
|
| 283 |
+
checkpoint_path: str | Path,
|
| 284 |
+
*,
|
| 285 |
+
device: str | torch.device,
|
| 286 |
+
dtype: torch.dtype = torch.bfloat16,
|
| 287 |
+
disable_numpy_memmap: bool = False,
|
| 288 |
+
nvfp4_scaled_mm: bool = False,
|
| 289 |
+
blocks_to_swap: int = 0,
|
| 290 |
+
attn_mode: str | None = None,
|
| 291 |
+
):
|
| 292 |
+
from transformers import Qwen3VLConfig, Qwen3VLModel
|
| 293 |
+
|
| 294 |
+
device = torch.device(device)
|
| 295 |
+
if blocks_to_swap > 0 and device.type != "cuda":
|
| 296 |
+
raise ValueError(
|
| 297 |
+
"--text_encoder_blocks_to_swap requires a CUDA device. / --text_encoder_blocks_to_swap には CUDA デバイスが必要です。"
|
| 298 |
+
)
|
| 299 |
+
|
| 300 |
+
config = Qwen3VLConfig.from_pretrained(H3_REPO_ID, subfolder=TEXT_ENCODER_CONFIG_SUBFOLDER)
|
| 301 |
+
if config.text_config.hidden_size != TEXT_WIDTH:
|
| 302 |
+
raise ValueError(f"MiniMax-H3 Qwen3-VL hidden size must be {TEXT_WIDTH}, got {config.text_config.hidden_size}")
|
| 303 |
+
config.text_config.num_hidden_layers = LAYER_50_HIDDEN_STATE_INDEX
|
| 304 |
+
config.text_config.use_cache = False
|
| 305 |
+
if attn_mode is not None:
|
| 306 |
+
# sdpa falls back to the O(L^2) math kernel at long context; flash_attention_2 is
|
| 307 |
+
# recommended for presentations beyond ~3k rows
|
| 308 |
+
config._attn_implementation = attn_mode
|
| 309 |
+
|
| 310 |
+
from accelerate import init_empty_weights
|
| 311 |
+
|
| 312 |
+
from musubi_tuner.modules.comfy_quant_utils import (
|
| 313 |
+
FORMAT_CONVROT_INT8,
|
| 314 |
+
FORMAT_INT8_TENSORWISE,
|
| 315 |
+
FORMAT_NVFP4,
|
| 316 |
+
detect_comfy_quant_formats,
|
| 317 |
+
)
|
| 318 |
+
from musubi_tuner.modules.convrot_int8_utils import ConvRotInt8Quantizer, apply_convrot_int8_monkey_patch
|
| 319 |
+
from musubi_tuner.modules.nvfp4_utils import NvFp4Quantizer, apply_nvfp4_monkey_patch
|
| 320 |
+
from musubi_tuner.utils.lora_utils import load_safetensors_with_lora_and_fp8
|
| 321 |
+
from musubi_tuner.utils.safetensors_utils import load_safetensors
|
| 322 |
+
|
| 323 |
+
with init_empty_weights():
|
| 324 |
+
model = Qwen3VLModel(config)
|
| 325 |
+
# the layer-50 pre-norm convention: the model is truncated to 50 layers and the final
|
| 326 |
+
# norm is replaced by Identity, so last_hidden_state IS the layer-50 pre-norm state
|
| 327 |
+
model.language_model.norm = nn.Identity()
|
| 328 |
+
|
| 329 |
+
# with block swap the state dict stays on the CPU and the streamed layers never fully
|
| 330 |
+
# reside on the device; otherwise loading straight to the target device avoids a
|
| 331 |
+
# resident full-model CPU copy
|
| 332 |
+
streaming = blocks_to_swap > 0
|
| 333 |
+
load_device = torch.device("cpu") if streaming else device
|
| 334 |
+
|
| 335 |
+
def _load_with_quantizer(quantizer):
|
| 336 |
+
# the same streaming loader as the transformer; the file dictates the quantized layers
|
| 337 |
+
sd = load_safetensors_with_lora_and_fp8(
|
| 338 |
+
model_files=[str(checkpoint_path)], # unexpanded: the loader expands split shards itself
|
| 339 |
+
lora_weights_list=None,
|
| 340 |
+
lora_multipliers=None,
|
| 341 |
+
fp8_optimization=False,
|
| 342 |
+
calc_device=device,
|
| 343 |
+
move_to_device=not streaming,
|
| 344 |
+
disable_numpy_memmap=disable_numpy_memmap,
|
| 345 |
+
quantizer=quantizer,
|
| 346 |
+
)
|
| 347 |
+
return {normalize_h3_text_encoder_key(key): value for key, value in sd.items()}
|
| 348 |
+
|
| 349 |
+
files = resolve_safetensors_files(checkpoint_path)
|
| 350 |
+
formats = detect_comfy_quant_formats(files, disable_numpy_memmap=disable_numpy_memmap)
|
| 351 |
+
if formats == {FORMAT_CONVROT_INT8}:
|
| 352 |
+
# pre-quantized ConvRot INT8 artifact; an empty target list disables dynamic quantization
|
| 353 |
+
quantizer = ConvRotInt8Quantizer(target_layer_keys=[])
|
| 354 |
+
sd = _load_with_quantizer(quantizer)
|
| 355 |
+
groupsize_map = {normalize_h3_text_encoder_key(key): value for key, value in quantizer.module_groupsizes.items()}
|
| 356 |
+
apply_convrot_int8_monkey_patch(model, sd, groupsize_map=groupsize_map)
|
| 357 |
+
# int8 tensors cannot require grad, and load_state_dict(assign=True) re-wraps incoming
|
| 358 |
+
# tensors with the meta params' requires_grad; the text encoder is frozen anyway
|
| 359 |
+
model.requires_grad_(False)
|
| 360 |
+
elif FORMAT_NVFP4 in formats and formats <= {FORMAT_NVFP4, FORMAT_INT8_TENSORWISE}:
|
| 361 |
+
# pre-quantized ComfyUI NVFP4 (+AWQ) artifact with an INT8 per-row embedding
|
| 362 |
+
quantizer = NvFp4Quantizer()
|
| 363 |
+
sd = _load_with_quantizer(quantizer)
|
| 364 |
+
nvfp4_module_shapes = {normalize_h3_text_encoder_key(key): value for key, value in quantizer.nvfp4_module_shapes.items()}
|
| 365 |
+
int8_embedding_modules = [normalize_h3_text_encoder_key(key) for key in quantizer.int8_embedding_modules]
|
| 366 |
+
apply_nvfp4_monkey_patch(
|
| 367 |
+
model, sd, nvfp4_module_shapes, int8_embedding_modules, use_scaled_mm=nvfp4_scaled_mm, embedding_dtype=dtype
|
| 368 |
+
)
|
| 369 |
+
model.requires_grad_(False)
|
| 370 |
+
elif formats:
|
| 371 |
+
raise ValueError(
|
| 372 |
+
f"Unsupported ComfyUI quantization format combination in text encoder checkpoint: {sorted(formats)}."
|
| 373 |
+
" Supported: ConvRot INT8, or NVFP4 with an INT8 embedding."
|
| 374 |
+
f" / テキストエンコーダの量子化形式の組み合わせ {sorted(formats)} はサポートされていません。"
|
| 375 |
+
"ConvRot INT8、または NVFP4(+INT8 embedding) のみ対応しています。"
|
| 376 |
+
)
|
| 377 |
+
else:
|
| 378 |
+
sd = {}
|
| 379 |
+
for file in files:
|
| 380 |
+
shard = load_safetensors(str(file), device=load_device, disable_mmap=True, disable_numpy_memmap=disable_numpy_memmap)
|
| 381 |
+
sd.update({normalize_h3_text_encoder_key(key): value for key, value in shard.items()})
|
| 382 |
+
|
| 383 |
+
# quantization scale tensors keep their own dtypes (fp32 row scales, fp8 block scales)
|
| 384 |
+
_KEEP_DTYPE_SUFFIXES = (".scale_weight", ".nvfp4_scale", ".nvfp4_block_scale")
|
| 385 |
+
for key in sd.keys():
|
| 386 |
+
if sd[key].is_floating_point() and not key.endswith(_KEEP_DTYPE_SUFFIXES):
|
| 387 |
+
sd[key] = sd[key].to(dtype)
|
| 388 |
+
model.load_state_dict(sd, strict=True, assign=True)
|
| 389 |
+
if streaming:
|
| 390 |
+
# the offloader requires frozen weights (the text encoder is frozen by contract anyway;
|
| 391 |
+
# the quantized branches have already dropped requires_grad, BF16 has not)
|
| 392 |
+
model.requires_grad_(False)
|
| 393 |
+
_enable_h3_text_encoder_streaming(model, device, blocks_to_swap)
|
| 394 |
+
else:
|
| 395 |
+
model.to(device)
|
| 396 |
+
model.eval()
|
| 397 |
+
return model
|
| 398 |
+
|
| 399 |
+
|
| 400 |
+
# quantization tensors that the ConvRot INT8 / NVFP4 monkey patches hang off the patched Linear
|
| 401 |
+
# modules; streamed together with the weight. An explicit allowlist keeps unrelated (tiny)
|
| 402 |
+
# buffers resident instead of silently streaming whatever a future patch registers.
|
| 403 |
+
_TE_STREAM_QUANT_BUFFER_NAMES = ("scale_weight", "nvfp4_block_scale", "nvfp4_scale", "pre_quant_scale")
|
| 404 |
+
|
| 405 |
+
|
| 406 |
+
def _te_swap_tensor_selector(block: nn.Module) -> list[tuple[nn.Module, str]]:
|
| 407 |
+
jobs: list[tuple[nn.Module, str]] = []
|
| 408 |
+
for _, module in block.named_modules():
|
| 409 |
+
if isinstance(module, nn.Linear) and module.weight is not None:
|
| 410 |
+
jobs.append((module, "weight"))
|
| 411 |
+
for name in _TE_STREAM_QUANT_BUFFER_NAMES:
|
| 412 |
+
if name in module._buffers:
|
| 413 |
+
jobs.append((module, name))
|
| 414 |
+
return jobs
|
| 415 |
+
|
| 416 |
+
|
| 417 |
+
def _enable_h3_text_encoder_streaming(model, device: torch.device, blocks_to_swap: int) -> None:
|
| 418 |
+
"""Stream ``blocks_to_swap`` of the 50 Qwen3-VL decoder layers from CPU masters (H2D only).
|
| 419 |
+
|
| 420 |
+
The text encoder is frozen and forward-only, so ``LoRAStreamOffloader`` applies as-is;
|
| 421 |
+
the transformers forward loop does not know about the offloader, so it is driven from
|
| 422 |
+
forward hooks. Everything outside the decoder layers (embeddings, vision tower, norms)
|
| 423 |
+
stays resident on the device. Masters are pageable (staged copier): no giant pinned
|
| 424 |
+
allocation, which matters on Windows.
|
| 425 |
+
"""
|
| 426 |
+
from musubi_tuner.modules.custom_offloading_utils import LoRAStreamOffloader, attach_forward_streaming_hooks
|
| 427 |
+
|
| 428 |
+
layers = list(_language_layers(model))
|
| 429 |
+
num_layers = len(layers)
|
| 430 |
+
if blocks_to_swap > num_layers:
|
| 431 |
+
raise ValueError(
|
| 432 |
+
f"--text_encoder_blocks_to_swap must be at most the number of text encoder layers ({num_layers}), got {blocks_to_swap}"
|
| 433 |
+
)
|
| 434 |
+
|
| 435 |
+
for name, child in model.named_children():
|
| 436 |
+
if name != "language_model":
|
| 437 |
+
child.to(device)
|
| 438 |
+
for name, child in model.language_model.named_children():
|
| 439 |
+
if name != "layers":
|
| 440 |
+
child.to(device)
|
| 441 |
+
|
| 442 |
+
offloader = LoRAStreamOffloader(
|
| 443 |
+
"mmh3-te",
|
| 444 |
+
layers,
|
| 445 |
+
num_layers,
|
| 446 |
+
blocks_to_swap,
|
| 447 |
+
supports_backward=False,
|
| 448 |
+
device=device,
|
| 449 |
+
ring_size=2,
|
| 450 |
+
use_pinned_memory=False,
|
| 451 |
+
swap_tensor_selector=_te_swap_tensor_selector,
|
| 452 |
+
)
|
| 453 |
+
# the handles are kept on the model so the hooks live exactly as long as the model does
|
| 454 |
+
model._h3_te_stream_hook_handles = attach_forward_streaming_hooks(offloader, layers)
|
| 455 |
+
offloader.prepare_block_devices_before_forward(layers)
|
| 456 |
+
model.h3_te_offloader = offloader
|
| 457 |
+
|
| 458 |
+
|
| 459 |
+
def _processor_value(value):
|
| 460 |
+
if isinstance(value, torch.Tensor):
|
| 461 |
+
return value.detach().cpu().numpy()
|
| 462 |
+
return np.asarray(value)
|
| 463 |
+
|
| 464 |
+
|
| 465 |
+
@torch.no_grad()
|
| 466 |
+
def encode_h3_presentation(processor, model, presentation: H3Presentation) -> tuple[torch.Tensor, torch.Tensor]:
|
| 467 |
+
processor_args = {
|
| 468 |
+
"text": [presentation.processor_text or presentation.text],
|
| 469 |
+
"return_tensors": "pt",
|
| 470 |
+
"padding": False,
|
| 471 |
+
}
|
| 472 |
+
if presentation.images:
|
| 473 |
+
processor_args["images"] = [_processor_value(image) for image in presentation.images]
|
| 474 |
+
if presentation.videos:
|
| 475 |
+
processor_args["videos"] = [_processor_value(video) for video in presentation.videos]
|
| 476 |
+
processor_args["video_metadata"] = [
|
| 477 |
+
{
|
| 478 |
+
"total_num_frames": int(video.shape[0]),
|
| 479 |
+
"fps": float(TEXT_VISUAL_FPS),
|
| 480 |
+
"duration": float(video.shape[0]) / TEXT_VISUAL_FPS,
|
| 481 |
+
"frames_indices": list(range(video.shape[0])),
|
| 482 |
+
"height": int(video.shape[1]),
|
| 483 |
+
"width": int(video.shape[2]),
|
| 484 |
+
}
|
| 485 |
+
for video in presentation.videos
|
| 486 |
+
]
|
| 487 |
+
processor_args["do_sample_frames"] = False
|
| 488 |
+
processed = processor(**processor_args)
|
| 489 |
+
token_tags = build_token_tags(processed)
|
| 490 |
+
row_count = token_tags.shape[0]
|
| 491 |
+
if row_count > MAX_TEXT_ROWS:
|
| 492 |
+
raise _text_size_error(row_count, token_tags)
|
| 493 |
+
|
| 494 |
+
layers = _language_layers(model)
|
| 495 |
+
if len(layers) != LAYER_50_HIDDEN_STATE_INDEX:
|
| 496 |
+
raise ValueError(f"Released MiniMax-H3 text encoder must contain exactly 50 layers, got {len(layers)}")
|
| 497 |
+
norm = getattr(model.language_model, "norm", None)
|
| 498 |
+
if not isinstance(norm, nn.Identity):
|
| 499 |
+
raise ValueError("MiniMax-H3 text encoder must have an Identity final norm (load via load_h3_text_encoder)")
|
| 500 |
+
|
| 501 |
+
# with block swap the layer weights live on CPU masters, so the input device is taken
|
| 502 |
+
# from the always-resident embedding
|
| 503 |
+
device = model.language_model.embed_tokens.weight.device
|
| 504 |
+
model_inputs = {
|
| 505 |
+
key: value.to(device) if isinstance(value, torch.Tensor) else value
|
| 506 |
+
for key, value in processed.items()
|
| 507 |
+
if key != "token_type_ids"
|
| 508 |
+
}
|
| 509 |
+
# the model is truncated to 50 layers and the final norm is Identity, so
|
| 510 |
+
# last_hidden_state is the layer-50 pre-norm hidden state
|
| 511 |
+
hidden_states = model(**model_inputs, use_cache=False).last_hidden_state
|
| 512 |
+
if hidden_states is None:
|
| 513 |
+
raise RuntimeError("MiniMax-H3 text encoder returned no last_hidden_state")
|
| 514 |
+
if hidden_states.ndim != 3 or hidden_states.shape[0] != 1:
|
| 515 |
+
raise ValueError(f"MiniMax-H3 layer-50 output must be [1,L,{TEXT_WIDTH}], got {tuple(hidden_states.shape)}")
|
| 516 |
+
hidden_states = hidden_states[0]
|
| 517 |
+
validate_text_rows(hidden_states, token_tags)
|
| 518 |
+
return hidden_states, token_tags
|
| 519 |
+
|
| 520 |
+
|
| 521 |
+
# The guidance-loss uncond cache: one text-only probe embedding (layer-50 hidden rows +
|
| 522 |
+
# token tags), shared between the cache script that writes it and the trainer that reads
|
| 523 |
+
# it. The format id matches the uncond-probe screening harness so screened probes load
|
| 524 |
+
# directly. Bump on any semantic change so stale caches are rejected.
|
| 525 |
+
UNCOND_CACHE_FORMAT = "h3-uncond-probe-v1"
|
| 526 |
+
|
| 527 |
+
|
| 528 |
+
def _validate_uncond_cache_tensors(hidden_states: torch.Tensor, token_tags: torch.Tensor, label: str) -> None:
|
| 529 |
+
if hidden_states.ndim != 2 or hidden_states.shape[0] < 1:
|
| 530 |
+
raise ValueError(f"MiniMax-H3 uncond cache {label} hidden states must be [L>=1,width], got {tuple(hidden_states.shape)}")
|
| 531 |
+
if hidden_states.shape[0] > MAX_TEXT_ROWS:
|
| 532 |
+
raise _text_size_error(hidden_states.shape[0], token_tags)
|
| 533 |
+
if token_tags.dtype != torch.int64 or token_tags.shape != (hidden_states.shape[0],):
|
| 534 |
+
raise ValueError(f"MiniMax-H3 uncond cache {label} token tags must be int64 [L]")
|
| 535 |
+
if not torch.all((token_tags == 0) | (token_tags == 1)):
|
| 536 |
+
raise ValueError(f"MiniMax-H3 uncond cache {label} token tags may contain only 0 and 1")
|
| 537 |
+
|
| 538 |
+
|
| 539 |
+
def save_h3_uncond_cache(
|
| 540 |
+
path: str | Path,
|
| 541 |
+
hidden_states: torch.Tensor,
|
| 542 |
+
token_tags: torch.Tensor,
|
| 543 |
+
*,
|
| 544 |
+
metadata: Mapping[str, str] | None = None,
|
| 545 |
+
) -> None:
|
| 546 |
+
from safetensors.torch import save_file
|
| 547 |
+
|
| 548 |
+
_validate_uncond_cache_tensors(hidden_states, token_tags, str(path))
|
| 549 |
+
path = Path(path)
|
| 550 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 551 |
+
save_file(
|
| 552 |
+
{"hidden_states": hidden_states.contiguous(), "token_tags": token_tags.contiguous()},
|
| 553 |
+
str(path),
|
| 554 |
+
metadata={"cache_format": UNCOND_CACHE_FORMAT, **dict(metadata or {})},
|
| 555 |
+
)
|
| 556 |
+
|
| 557 |
+
|
| 558 |
+
def load_h3_uncond_cache(path: str | Path) -> tuple[torch.Tensor, torch.Tensor, dict[str, str]]:
|
| 559 |
+
from safetensors import safe_open
|
| 560 |
+
|
| 561 |
+
path = Path(path)
|
| 562 |
+
with safe_open(str(path), framework="pt", device="cpu") as handle:
|
| 563 |
+
metadata = dict(handle.metadata() or {})
|
| 564 |
+
cached_format = metadata.get("cache_format")
|
| 565 |
+
if cached_format != UNCOND_CACHE_FORMAT:
|
| 566 |
+
raise ValueError(f"MiniMax-H3 uncond cache format must be {UNCOND_CACHE_FORMAT!r}, got {cached_format!r}: {path}")
|
| 567 |
+
if set(handle.keys()) != {"hidden_states", "token_tags"}:
|
| 568 |
+
raise ValueError(f"MiniMax-H3 uncond cache has an invalid tensor-key set: {path}")
|
| 569 |
+
hidden_states = handle.get_tensor("hidden_states")
|
| 570 |
+
token_tags = handle.get_tensor("token_tags")
|
| 571 |
+
_validate_uncond_cache_tensors(hidden_states, token_tags, str(path))
|
| 572 |
+
return hidden_states, token_tags, metadata
|
| 573 |
+
|
| 574 |
+
|
| 575 |
+
def processor_fingerprint(processor) -> str:
|
| 576 |
+
tokenizer = getattr(processor, "tokenizer", None)
|
| 577 |
+
payload = {
|
| 578 |
+
"processor_class": type(processor).__name__,
|
| 579 |
+
"processor_name": getattr(processor, "name_or_path", None),
|
| 580 |
+
"tokenizer_class": type(tokenizer).__name__ if tokenizer is not None else None,
|
| 581 |
+
"tokenizer_name": getattr(tokenizer, "name_or_path", None),
|
| 582 |
+
"tokenizer_size": len(tokenizer) if tokenizer is not None else None,
|
| 583 |
+
"commit": getattr(processor, "_commit_hash", None) or (getattr(tokenizer, "init_kwargs", {}) or {}).get("_commit_hash"),
|
| 584 |
+
"vision_tokens": [VISION_START_TOKEN_ID, VISION_END_TOKEN_ID, IMAGE_TOKEN_ID, VIDEO_TOKEN_ID],
|
| 585 |
+
}
|
| 586 |
+
encoded = json.dumps(payload, sort_keys=True, ensure_ascii=True, separators=(",", ":")).encode("utf-8")
|
| 587 |
+
return f"sha256:{hashlib.sha256(encoded).hexdigest()}"
|
| 588 |
+
|
| 589 |
+
|
| 590 |
+
# Bump whenever the cached hidden-state semantics change (hidden-state convention, token-tag
|
| 591 |
+
# algorithm, text layout constants, or the fingerprint formats) so stale caches are rebuilt/rejected.
|
| 592 |
+
TEXT_CACHE_FORMAT = "minimax-h3-text-v2"
|
| 593 |
+
|
| 594 |
+
# Teacher-matching condition seam. "first,last" conditions the FL2VA teacher on both endpoints
|
| 595 |
+
# (full training videos always provide them, and the FL2VA base is most in-distribution with
|
| 596 |
+
# both anchors). "ref" conditions the teacher on the training clip itself through the Ref2VA
|
| 597 |
+
# layout (reference video plus its audio track), giving the teacher complete information at
|
| 598 |
+
# every sigma. The value is validated through this seam so further variants (single-sided,
|
| 599 |
+
# anchored, segmented teachers) can slot in later without changing the cache or trainer
|
| 600 |
+
# interfaces.
|
| 601 |
+
TEACHER_CONDITIONS_FIRST_LAST = "first,last"
|
| 602 |
+
TEACHER_CONDITIONS_REF = "ref"
|
| 603 |
+
TEACHER_CONDITIONS_SUBJECT_REF = "subject_ref"
|
| 604 |
+
|
| 605 |
+
# key stem of the teacher text rows a text cache may carry next to the student rows, per teacher
|
| 606 |
+
# kind: each kind uses distinct keys so the trainer hard-fails on a cache/flag mode mismatch
|
| 607 |
+
# instead of silently misreading the rows (the cache writer appends `_hidden_states_<dtype>` /
|
| 608 |
+
# `_token_tags_int64`; the training collator drops the `varlen_` marker and the dtype suffix)
|
| 609 |
+
TEACHER_TEXT_CACHE_PREFIXES = {
|
| 610 |
+
TEACHER_CONDITIONS_FIRST_LAST: "varlen_mmh3_teacher",
|
| 611 |
+
TEACHER_CONDITIONS_REF: "varlen_mmh3_teacher_ref",
|
| 612 |
+
TEACHER_CONDITIONS_SUBJECT_REF: "varlen_mmh3_teacher_subject_ref",
|
| 613 |
+
}
|
| 614 |
+
|
| 615 |
+
|
| 616 |
+
def normalize_teacher_conditions(value: str) -> str:
|
| 617 |
+
parts = [part.strip() for part in str(value).split(",")]
|
| 618 |
+
if parts == ["first", "last"]:
|
| 619 |
+
return TEACHER_CONDITIONS_FIRST_LAST
|
| 620 |
+
if parts == [TEACHER_CONDITIONS_REF]:
|
| 621 |
+
return TEACHER_CONDITIONS_REF
|
| 622 |
+
if parts == [TEACHER_CONDITIONS_SUBJECT_REF]:
|
| 623 |
+
return TEACHER_CONDITIONS_SUBJECT_REF
|
| 624 |
+
raise ValueError(
|
| 625 |
+
f"MiniMax-H3 teacher matching supports only teacher conditions "
|
| 626 |
+
f"'{TEACHER_CONDITIONS_FIRST_LAST}', '{TEACHER_CONDITIONS_REF}' or '{TEACHER_CONDITIONS_SUBJECT_REF}', got {value!r}"
|
| 627 |
+
)
|
| 628 |
+
|
| 629 |
+
|
| 630 |
+
# The subject-reference teacher caption wrap: the official full-reference declaration blocks
|
| 631 |
+
# that make the base read each picture as a *subject* (identity/appearance) reference rather
|
| 632 |
+
# than a frame of the target. Modeled on the presentation that extracted a character's
|
| 633 |
+
# identity on the released FL2VA weights (probe A, 2026-09-02) with the appearance clauses
|
| 634 |
+
# removed, so it is content-independent boilerplate around the user caption like the
|
| 635 |
+
# ref-teacher wrap; `attribute_transfer` is the empirically validated marker and the
|
| 636 |
+
# "pose, framing, outfit and setting follow the description" clause is what keeps the
|
| 637 |
+
# picture from acting as a copy source.
|
| 638 |
+
SUBJECT_REF_SUMMARY_IMAGE = "The target is a single still image with no motion, a static shot of {subjects} as described below."
|
| 639 |
+
SUBJECT_REF_SUMMARY_VIDEO = "The target video shows {subjects} as described below."
|
| 640 |
+
|
| 641 |
+
|
| 642 |
+
def wrap_subject_reference_caption(caption: str, image_count: int, *, still_image: bool) -> str:
|
| 643 |
+
if image_count < 1:
|
| 644 |
+
raise ValueError("MiniMax-H3 subject-reference caption requires at least one picture")
|
| 645 |
+
labels = [f"<Subject {index}>" for index in range(1, image_count + 1)]
|
| 646 |
+
if len(labels) == 1:
|
| 647 |
+
subjects = labels[0]
|
| 648 |
+
else:
|
| 649 |
+
subjects = ", ".join(labels[:-1]) + f" and {labels[-1]}"
|
| 650 |
+
definitions = "\n".join(
|
| 651 |
+
f"<Subject {index}> is the subject whose appearance comes from <Picture {index}> (face and hair style)."
|
| 652 |
+
for index in range(1, image_count + 1)
|
| 653 |
+
)
|
| 654 |
+
summary = (SUBJECT_REF_SUMMARY_IMAGE if still_image else SUBJECT_REF_SUMMARY_VIDEO).format(subjects=subjects)
|
| 655 |
+
retention = "\n".join(
|
| 656 |
+
f"<Subject {index}> (appears in [Shot 1]): attribute_transfer - the appearance of <Subject {index}> in"
|
| 657 |
+
f" <Picture {index}> is referenced; pose, framing, outfit and setting follow the description."
|
| 658 |
+
for index in range(1, image_count + 1)
|
| 659 |
+
)
|
| 660 |
+
return (
|
| 661 |
+
f"subject_definitions:\n{definitions}\n\n"
|
| 662 |
+
f"summary:\n[reference generation] {summary}\n\n"
|
| 663 |
+
f"retention_analysis:\n{retention}\n\n"
|
| 664 |
+
f"detailed_description:\n{caption}"
|
| 665 |
+
)
|
| 666 |
+
|
| 667 |
+
|
| 668 |
+
# The ref-teacher caption wrap: the official editing-prompt declaration blocks that make the
|
| 669 |
+
# base model treat the reference as a 1:1 copy source. subject_definitions / summary /
|
| 670 |
+
# retention_analysis are content-independent boilerplate, so the cache script wraps the user
|
| 671 |
+
# caption automatically, like the FL2VA Picture prefix. Probe-validated on the released FL2VA
|
| 672 |
+
# weights: the video copy semantics saturate even without any declaration, but the
|
| 673 |
+
# `<Audio 1>: fully_copy` declaration is what opens audio education across the teaching band.
|
| 674 |
+
REF_TEACHER_CAPTION_HEADER = """subject_definitions:
|
| 675 |
+
<Video 1> is the source video for the target video edit.
|
| 676 |
+
<Audio 1> is the synchronized audio track of <Video 1> and is reused in the target video.
|
| 677 |
+
|
| 678 |
+
summary:
|
| 679 |
+
[video editing + audio reuse] The target video is an edited version of <Video 1> with no changes; all shots, subjects, camera movement, and sound are preserved as they are.
|
| 680 |
+
|
| 681 |
+
retention_analysis:
|
| 682 |
+
<Video 1> (all shots): fully_preserved - every shot, subject, action, and camera movement of the source video is retained without modification.
|
| 683 |
+
<Audio 1>: fully_copy - <Audio 1> is reused 1:1 as the target video's complete final audio track.
|
| 684 |
+
|
| 685 |
+
detailed_description:
|
| 686 |
+
"""
|
| 687 |
+
|
| 688 |
+
|
| 689 |
+
def wrap_ref_teacher_caption(caption: str) -> str:
|
| 690 |
+
return REF_TEACHER_CAPTION_HEADER + caption
|
| 691 |
+
|
| 692 |
+
|
| 693 |
+
def presentation_fingerprint(
|
| 694 |
+
presentation: H3Presentation,
|
| 695 |
+
media_fingerprints: Mapping[Path, str],
|
| 696 |
+
*,
|
| 697 |
+
frame_count: int,
|
| 698 |
+
) -> str:
|
| 699 |
+
# frame_count is part of the identity because Ref2VA reference videos are resampled to the
|
| 700 |
+
# target frame count before presentation. The target crop start is deliberately excluded: it
|
| 701 |
+
# only affects FL2VA visuals, and is guarded by explicit cache metadata instead.
|
| 702 |
+
payload = {
|
| 703 |
+
"text": presentation.text,
|
| 704 |
+
"processor_text": presentation.processor_text,
|
| 705 |
+
"image_shapes": [list(image.shape) for image in presentation.images],
|
| 706 |
+
"video_shapes": [list(video.shape) for video in presentation.videos],
|
| 707 |
+
"media": dict(sorted((str(Path(path).resolve()), value) for path, value in media_fingerprints.items())),
|
| 708 |
+
"frame_count": frame_count,
|
| 709 |
+
"format": "minimax-h3-non-chat-v2",
|
| 710 |
+
}
|
| 711 |
+
encoded = json.dumps(payload, sort_keys=True, ensure_ascii=True, separators=(",", ":")).encode("utf-8")
|
| 712 |
+
return f"sha256:{hashlib.sha256(encoded).hexdigest()}"
|
third_party/musubi_tuner/minimax_h3/video_vae.py
ADDED
|
@@ -0,0 +1,727 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2026 The MiniMax and HuggingFace Teams. All rights reserved.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
#
|
| 15 |
+
# Adapted for Musubi from Hugging Face Diffusers PR #14355 at commit
|
| 16 |
+
# abc5e9bf71fd38f53cd471bc3acaa84bc5ecbfdc. Musubi keeps the published
|
| 17 |
+
# checkpoint names and adds cache-specific posterior sampling wrappers.
|
| 18 |
+
# ComfyUI is used only as an independent numerical reference.
|
| 19 |
+
|
| 20 |
+
import hashlib
|
| 21 |
+
import math
|
| 22 |
+
|
| 23 |
+
import torch
|
| 24 |
+
import torch.nn as nn
|
| 25 |
+
import torch.nn.functional as F
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
IMAGENET_MEAN = (0.485, 0.456, 0.406)
|
| 29 |
+
IMAGENET_STD = (0.229, 0.224, 0.225)
|
| 30 |
+
VIDEO_VAE_ENCODE_DTYPE = torch.float32
|
| 31 |
+
VIDEO_VAE_DECODE_DTYPE = torch.float16
|
| 32 |
+
# State-dict prefixes of the ViT decoder, which encode-only users (latent caching) leave unloaded:
|
| 33 |
+
# the decoder holds ~2.4B of the VAE's ~2.6B parameters.
|
| 34 |
+
VIDEO_VAE_DECODER_KEY_PREFIXES = ("decoder.", "post_quant_conv.")
|
| 35 |
+
|
| 36 |
+
LATENTS_MEAN = [
|
| 37 |
+
0.858090341091156,
|
| 38 |
+
-0.9606591463088989,
|
| 39 |
+
1.0661640167236328,
|
| 40 |
+
-0.5090325474739075,
|
| 41 |
+
-0.2727581858634949,
|
| 42 |
+
-1.3675414323806763,
|
| 43 |
+
-0.2553254961967468,
|
| 44 |
+
-0.26907554268836975,
|
| 45 |
+
-0.5376840829849243,
|
| 46 |
+
-0.0464097298681736,
|
| 47 |
+
0.6657370328903198,
|
| 48 |
+
0.19690127670764923,
|
| 49 |
+
-0.5460608005523682,
|
| 50 |
+
-0.4035342037677765,
|
| 51 |
+
-0.23683024942874908,
|
| 52 |
+
0.25928452610969543,
|
| 53 |
+
-0.30133944749832153,
|
| 54 |
+
0.211341992020607,
|
| 55 |
+
-1.1206848621368408,
|
| 56 |
+
0.3581933379173279,
|
| 57 |
+
-0.04225143790245056,
|
| 58 |
+
0.2604829967021942,
|
| 59 |
+
0.22864092886447906,
|
| 60 |
+
0.7056031823158264,
|
| 61 |
+
]
|
| 62 |
+
|
| 63 |
+
LATENTS_STD = [
|
| 64 |
+
1.2223774194717407,
|
| 65 |
+
1.2767263650894165,
|
| 66 |
+
1.68317747116088865,
|
| 67 |
+
1.7549455165863037,
|
| 68 |
+
1.5636216402053833,
|
| 69 |
+
2.194143533706665,
|
| 70 |
+
0.96531379222869875,
|
| 71 |
+
1.05698859691619875,
|
| 72 |
+
0.841948926448822,
|
| 73 |
+
0.7729952931404114,
|
| 74 |
+
1.8955937623977661,
|
| 75 |
+
0.946841835975647,
|
| 76 |
+
0.7996809482574463,
|
| 77 |
+
0.44988900423049925,
|
| 78 |
+
0.7197399735450745,
|
| 79 |
+
0.69362932443618775,
|
| 80 |
+
2.961095094680786,
|
| 81 |
+
2.7694199085235595,
|
| 82 |
+
3.0496184825897215,
|
| 83 |
+
2.1088054180145265,
|
| 84 |
+
3.276226282119751,
|
| 85 |
+
3.1627357006073,
|
| 86 |
+
2.28168129920959475,
|
| 87 |
+
2.6127843856811525,
|
| 88 |
+
]
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
class CausalConv3d(nn.Conv3d):
|
| 92 |
+
"""Spatially reflected, temporally causal 3D convolution."""
|
| 93 |
+
|
| 94 |
+
def __init__(self, in_channels, out_channels, kernel_size, stride=1, padding=0):
|
| 95 |
+
super().__init__(in_channels, out_channels, kernel_size=kernel_size, stride=stride, padding=0)
|
| 96 |
+
self.causal_padding = (padding,) * 3 if isinstance(padding, int) else tuple(padding)
|
| 97 |
+
|
| 98 |
+
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
| 99 |
+
temporal, height, width = self.causal_padding
|
| 100 |
+
if height or width:
|
| 101 |
+
hidden_states = F.pad(hidden_states, (width, width, height, height, 0, 0), mode="reflect")
|
| 102 |
+
if temporal:
|
| 103 |
+
hidden_states = F.pad(hidden_states, (0, 0, 0, 0, temporal * 2, 0), mode="constant")
|
| 104 |
+
return F.conv3d(
|
| 105 |
+
hidden_states,
|
| 106 |
+
self.weight,
|
| 107 |
+
self.bias,
|
| 108 |
+
stride=self.stride,
|
| 109 |
+
padding=0,
|
| 110 |
+
dilation=self.dilation,
|
| 111 |
+
groups=self.groups,
|
| 112 |
+
)
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
class TemporalIsolatedGroupNorm(nn.GroupNorm):
|
| 116 |
+
"""Compute group-normalization statistics independently for each frame."""
|
| 117 |
+
|
| 118 |
+
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
| 119 |
+
if hidden_states.ndim != 5:
|
| 120 |
+
return super().forward(hidden_states)
|
| 121 |
+
batch_size, channels, frames, height, width = hidden_states.shape
|
| 122 |
+
hidden_states = hidden_states.permute(0, 2, 1, 3, 4).contiguous()
|
| 123 |
+
hidden_states = hidden_states.view(batch_size * frames, channels, 1, height, width)
|
| 124 |
+
hidden_states = super().forward(hidden_states)
|
| 125 |
+
hidden_states = hidden_states.view(batch_size, frames, channels, height, width)
|
| 126 |
+
return hidden_states.permute(0, 2, 1, 3, 4).contiguous()
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
def group_norm_3d(num_channels: int) -> TemporalIsolatedGroupNorm:
|
| 130 |
+
return TemporalIsolatedGroupNorm(32, num_channels, eps=1e-6, affine=True)
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
class Downsample3D(nn.Module):
|
| 134 |
+
def __init__(self, in_channels, out_channels, time_stride=1, space_stride=2):
|
| 135 |
+
super().__init__()
|
| 136 |
+
self.space_stride = space_stride
|
| 137 |
+
self.conv = CausalConv3d(
|
| 138 |
+
in_channels,
|
| 139 |
+
out_channels,
|
| 140 |
+
kernel_size=3,
|
| 141 |
+
stride=(time_stride, space_stride, space_stride),
|
| 142 |
+
padding=(1, 0, 0),
|
| 143 |
+
)
|
| 144 |
+
|
| 145 |
+
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
| 146 |
+
if self.space_stride == 2:
|
| 147 |
+
hidden_states = F.pad(hidden_states, (0, 1, 0, 1, 0, 0), mode="reflect")
|
| 148 |
+
return self.conv(hidden_states)
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
class ResnetBlock3D(nn.Module):
|
| 152 |
+
def __init__(self, in_channels, out_channels=None):
|
| 153 |
+
super().__init__()
|
| 154 |
+
self.in_channels = in_channels
|
| 155 |
+
self.out_channels = in_channels if out_channels is None else out_channels
|
| 156 |
+
self.norm1 = group_norm_3d(in_channels)
|
| 157 |
+
self.conv1 = CausalConv3d(in_channels, self.out_channels, kernel_size=3, padding=1)
|
| 158 |
+
self.norm2 = group_norm_3d(self.out_channels)
|
| 159 |
+
self.conv2 = CausalConv3d(self.out_channels, self.out_channels, kernel_size=3, padding=1)
|
| 160 |
+
self.nin_shortcut = (
|
| 161 |
+
CausalConv3d(in_channels, self.out_channels, kernel_size=1) if in_channels != self.out_channels else None
|
| 162 |
+
)
|
| 163 |
+
|
| 164 |
+
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
| 165 |
+
residual = hidden_states
|
| 166 |
+
hidden_states = self.conv1(F.silu(self.norm1(hidden_states)))
|
| 167 |
+
hidden_states = self.conv2(F.silu(self.norm2(hidden_states)))
|
| 168 |
+
if self.nin_shortcut is not None:
|
| 169 |
+
residual = self.nin_shortcut(residual)
|
| 170 |
+
return residual + hidden_states
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
class EncoderFCN3D(nn.Module):
|
| 174 |
+
def __init__(self, ch, ch_mult, space_down, time_down, num_res_blocks, in_channels, z_channels, double_z=True):
|
| 175 |
+
super().__init__()
|
| 176 |
+
num_levels = len(ch_mult)
|
| 177 |
+
layers_per_level = [num_res_blocks] * num_levels if isinstance(num_res_blocks, int) else list(num_res_blocks)
|
| 178 |
+
block_channels = [ch * multiplier for multiplier in ch_mult]
|
| 179 |
+
input_channels = [block_channels[0], *block_channels[:-1]]
|
| 180 |
+
|
| 181 |
+
self.num_levels = num_levels
|
| 182 |
+
self.num_res_blocks = layers_per_level
|
| 183 |
+
self.conv_in = CausalConv3d(in_channels, input_channels[0], kernel_size=3, padding=1)
|
| 184 |
+
self.down = nn.ModuleList()
|
| 185 |
+
for level in range(num_levels):
|
| 186 |
+
down = nn.Module()
|
| 187 |
+
down.block = nn.ModuleList(
|
| 188 |
+
[
|
| 189 |
+
ResnetBlock3D(
|
| 190 |
+
input_channels[level] if layer == 0 else block_channels[level],
|
| 191 |
+
block_channels[level],
|
| 192 |
+
)
|
| 193 |
+
for layer in range(layers_per_level[level])
|
| 194 |
+
]
|
| 195 |
+
)
|
| 196 |
+
if space_down[level] * time_down[level] > 1:
|
| 197 |
+
down.downsample = Downsample3D(
|
| 198 |
+
block_channels[level],
|
| 199 |
+
block_channels[level],
|
| 200 |
+
time_stride=time_down[level],
|
| 201 |
+
space_stride=space_down[level],
|
| 202 |
+
)
|
| 203 |
+
self.down.append(down)
|
| 204 |
+
|
| 205 |
+
self.norm_out = group_norm_3d(block_channels[-1])
|
| 206 |
+
output_channels = 2 * z_channels if double_z else z_channels
|
| 207 |
+
self.conv_out = CausalConv3d(block_channels[-1], output_channels, kernel_size=3, padding=1)
|
| 208 |
+
|
| 209 |
+
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
| 210 |
+
hidden_states = self.conv_in(hidden_states)
|
| 211 |
+
for down in self.down:
|
| 212 |
+
for block in down.block:
|
| 213 |
+
hidden_states = block(hidden_states)
|
| 214 |
+
if hasattr(down, "downsample"):
|
| 215 |
+
hidden_states = down.downsample(hidden_states)
|
| 216 |
+
hidden_states = F.silu(self.norm_out(hidden_states))
|
| 217 |
+
return self.conv_out(hidden_states)
|
| 218 |
+
|
| 219 |
+
|
| 220 |
+
def create_token_ids(patch_dims, device, dtype=torch.float32):
|
| 221 |
+
grids = [2.0 * (torch.arange(0.5, size, dtype=dtype, device=device) / size) - 1.0 for size in patch_dims]
|
| 222 |
+
return torch.stack(torch.meshgrid(*grids, indexing="ij"), dim=-1).flatten(0, len(patch_dims) - 1).unsqueeze(0)
|
| 223 |
+
|
| 224 |
+
|
| 225 |
+
class RotaryEmbeddingND(nn.Module):
|
| 226 |
+
def __init__(self, dim, rotary_base=100.0, n_dim=3):
|
| 227 |
+
super().__init__()
|
| 228 |
+
if dim % (2 * n_dim):
|
| 229 |
+
raise ValueError(f"Rotary dimension {dim} must be divisible by {2 * n_dim}")
|
| 230 |
+
inv_freq = 1.0 / rotary_base ** torch.arange(0, 1, 2 * n_dim / dim, dtype=torch.float32)
|
| 231 |
+
self.register_buffer("inv_freq", inv_freq, persistent=False)
|
| 232 |
+
|
| 233 |
+
def forward(self, position_ids: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
| 234 |
+
angles = 2.0 * math.pi * position_ids[:, :, :, None].float() * self.inv_freq[None, None, None, :]
|
| 235 |
+
angles = angles.flatten(2, 3).tile(2).unsqueeze(2)
|
| 236 |
+
return angles.cos(), angles.sin()
|
| 237 |
+
|
| 238 |
+
|
| 239 |
+
def _apply_rotary_emb(hidden_states: torch.Tensor, rotary_emb: tuple[torch.Tensor, torch.Tensor]) -> torch.Tensor:
|
| 240 |
+
cosine, sine = (value.to(hidden_states.dtype) for value in rotary_emb)
|
| 241 |
+
rotary_dim = cosine.shape[-1]
|
| 242 |
+
rotary, passthrough = hidden_states[..., :rotary_dim], hidden_states[..., rotary_dim:]
|
| 243 |
+
first, second = rotary.chunk(2, dim=-1)
|
| 244 |
+
rotated = torch.cat((-second, first), dim=-1)
|
| 245 |
+
return torch.cat((rotary * cosine + rotated * sine, passthrough), dim=-1)
|
| 246 |
+
|
| 247 |
+
|
| 248 |
+
class FeedForward(nn.Module):
|
| 249 |
+
def __init__(self, dim, mult=4, bias=True):
|
| 250 |
+
super().__init__()
|
| 251 |
+
inner_dim = dim * mult
|
| 252 |
+
self.w1 = nn.Linear(dim, inner_dim * 2, bias=bias)
|
| 253 |
+
self.w2 = nn.Linear(inner_dim, dim, bias=bias)
|
| 254 |
+
|
| 255 |
+
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
| 256 |
+
gate, value = self.w1(hidden_states).chunk(2, dim=-1)
|
| 257 |
+
return self.w2(F.silu(gate) * value)
|
| 258 |
+
|
| 259 |
+
|
| 260 |
+
class Attention(nn.Module):
|
| 261 |
+
def __init__(self, heads, dim_head, bias=True, eps=1e-5):
|
| 262 |
+
super().__init__()
|
| 263 |
+
self.dim_head = dim_head
|
| 264 |
+
self.heads = heads
|
| 265 |
+
inner_dim = heads * dim_head
|
| 266 |
+
self.norm_q = nn.RMSNorm(dim_head, eps=eps, elementwise_affine=False)
|
| 267 |
+
self.norm_k = nn.RMSNorm(dim_head, eps=eps, elementwise_affine=False)
|
| 268 |
+
self.to_qkv = nn.Linear(inner_dim, inner_dim * 3, bias=bias)
|
| 269 |
+
self.to_out = nn.Linear(inner_dim, inner_dim, bias=bias)
|
| 270 |
+
|
| 271 |
+
def forward(
|
| 272 |
+
self,
|
| 273 |
+
hidden_states: torch.Tensor,
|
| 274 |
+
rotary_emb: tuple[torch.Tensor, torch.Tensor] | None = None,
|
| 275 |
+
) -> torch.Tensor:
|
| 276 |
+
batch_size, sequence_length, _ = hidden_states.shape
|
| 277 |
+
qkv = self.to_qkv(hidden_states).view(batch_size, sequence_length, self.heads, 3 * self.dim_head)
|
| 278 |
+
query, key, value = qkv.chunk(3, dim=-1)
|
| 279 |
+
query = self.norm_q(query.float()).to(query.dtype)
|
| 280 |
+
key = self.norm_k(key.float()).to(key.dtype)
|
| 281 |
+
if rotary_emb is not None:
|
| 282 |
+
query = _apply_rotary_emb(query, rotary_emb)
|
| 283 |
+
key = _apply_rotary_emb(key, rotary_emb)
|
| 284 |
+
hidden_states = F.scaled_dot_product_attention(
|
| 285 |
+
query.transpose(1, 2),
|
| 286 |
+
key.transpose(1, 2),
|
| 287 |
+
value.transpose(1, 2),
|
| 288 |
+
)
|
| 289 |
+
hidden_states = hidden_states.transpose(1, 2).reshape(batch_size, sequence_length, -1)
|
| 290 |
+
return self.to_out(hidden_states)
|
| 291 |
+
|
| 292 |
+
|
| 293 |
+
class TransformerBlock(nn.Module):
|
| 294 |
+
def __init__(self, heads, dim_head, bias=True, eps=1e-5):
|
| 295 |
+
super().__init__()
|
| 296 |
+
dim = heads * dim_head
|
| 297 |
+
self.norm1 = nn.RMSNorm(dim, elementwise_affine=True, eps=eps)
|
| 298 |
+
self.attn = Attention(heads=heads, dim_head=dim_head, bias=bias, eps=eps)
|
| 299 |
+
self.scale1 = nn.Parameter(torch.empty(dim))
|
| 300 |
+
self.norm2 = nn.RMSNorm(dim, elementwise_affine=True, eps=eps)
|
| 301 |
+
self.ff = FeedForward(dim=dim, bias=bias)
|
| 302 |
+
self.scale2 = nn.Parameter(torch.empty(dim))
|
| 303 |
+
|
| 304 |
+
def forward(
|
| 305 |
+
self,
|
| 306 |
+
hidden_states: torch.Tensor,
|
| 307 |
+
rotary_emb: tuple[torch.Tensor, torch.Tensor] | None = None,
|
| 308 |
+
) -> torch.Tensor:
|
| 309 |
+
normalized = F.rms_norm(
|
| 310 |
+
hidden_states.float(),
|
| 311 |
+
(hidden_states.shape[-1],),
|
| 312 |
+
self.norm1.weight.float(),
|
| 313 |
+
self.norm1.eps,
|
| 314 |
+
).to(hidden_states.dtype)
|
| 315 |
+
hidden_states = hidden_states + self.attn(normalized, rotary_emb) * self.scale1
|
| 316 |
+
normalized = F.rms_norm(
|
| 317 |
+
hidden_states.float(),
|
| 318 |
+
(hidden_states.shape[-1],),
|
| 319 |
+
self.norm2.weight.float(),
|
| 320 |
+
self.norm2.eps,
|
| 321 |
+
).to(hidden_states.dtype)
|
| 322 |
+
return hidden_states + self.ff(normalized) * self.scale2
|
| 323 |
+
|
| 324 |
+
|
| 325 |
+
class ViT3DDecoder(nn.Module):
|
| 326 |
+
def __init__(
|
| 327 |
+
self,
|
| 328 |
+
patch_size=16,
|
| 329 |
+
patch_size_t=4,
|
| 330 |
+
in_channels=24,
|
| 331 |
+
out_channels=3,
|
| 332 |
+
num_layers=36,
|
| 333 |
+
heads=32,
|
| 334 |
+
dim_head=64,
|
| 335 |
+
rope_theta=100.0,
|
| 336 |
+
rope_dim_ratio=0.75,
|
| 337 |
+
bias=True,
|
| 338 |
+
eps=1e-5,
|
| 339 |
+
num_register_tokens=4,
|
| 340 |
+
):
|
| 341 |
+
super().__init__()
|
| 342 |
+
dim = heads * dim_head
|
| 343 |
+
self.patch_size = patch_size
|
| 344 |
+
self.patch_size_t = patch_size_t
|
| 345 |
+
self.out_channels = out_channels
|
| 346 |
+
self.num_register_tokens = num_register_tokens
|
| 347 |
+
self.pos_embed = RotaryEmbeddingND(int(dim_head * rope_dim_ratio), rope_theta, n_dim=3)
|
| 348 |
+
self.x_embedder = nn.Linear(in_channels, dim)
|
| 349 |
+
self.register_tokens = nn.Parameter(torch.empty(1, num_register_tokens, dim))
|
| 350 |
+
self.register_buffer("mask_token", torch.empty(1, 1, dim))
|
| 351 |
+
self.transformer_blocks = nn.ModuleList(
|
| 352 |
+
[TransformerBlock(heads=heads, dim_head=dim_head, bias=bias, eps=eps) for _ in range(num_layers)]
|
| 353 |
+
)
|
| 354 |
+
self.norm_out = nn.LayerNorm(dim, elementwise_affine=True, eps=eps)
|
| 355 |
+
self.proj_out = nn.Linear(dim, out_channels * patch_size_t * patch_size * patch_size)
|
| 356 |
+
|
| 357 |
+
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
| 358 |
+
batch_size, channels, frames, height, width = hidden_states.shape
|
| 359 |
+
hidden_states = hidden_states.permute(0, 2, 3, 4, 1).reshape(batch_size, frames * height * width, channels)
|
| 360 |
+
hidden_states = self.x_embedder(hidden_states)
|
| 361 |
+
num_patches = hidden_states.shape[1]
|
| 362 |
+
hidden_states = torch.cat(
|
| 363 |
+
(
|
| 364 |
+
hidden_states,
|
| 365 |
+
self.register_tokens.expand(batch_size, -1, -1),
|
| 366 |
+
torch.zeros_like(hidden_states[:, :1]),
|
| 367 |
+
),
|
| 368 |
+
dim=1,
|
| 369 |
+
)
|
| 370 |
+
|
| 371 |
+
position_ids = create_token_ids((frames, height, width), hidden_states.device)
|
| 372 |
+
position_ids = position_ids.expand(batch_size, -1, -1)
|
| 373 |
+
suffix_ids = position_ids.new_zeros((batch_size, self.num_register_tokens + 1, 3))
|
| 374 |
+
rotary_emb = self.pos_embed(torch.cat((position_ids, suffix_ids), dim=1))
|
| 375 |
+
for block in self.transformer_blocks:
|
| 376 |
+
hidden_states = block(hidden_states, rotary_emb)
|
| 377 |
+
|
| 378 |
+
hidden_states = self.proj_out(self.norm_out(hidden_states))[:, :num_patches]
|
| 379 |
+
hidden_states = hidden_states.view(
|
| 380 |
+
batch_size,
|
| 381 |
+
frames,
|
| 382 |
+
height,
|
| 383 |
+
width,
|
| 384 |
+
self.out_channels,
|
| 385 |
+
self.patch_size_t,
|
| 386 |
+
self.patch_size,
|
| 387 |
+
self.patch_size,
|
| 388 |
+
)
|
| 389 |
+
hidden_states = hidden_states.permute(0, 4, 1, 5, 2, 6, 3, 7).contiguous()
|
| 390 |
+
return hidden_states.reshape(
|
| 391 |
+
batch_size,
|
| 392 |
+
self.out_channels,
|
| 393 |
+
frames * self.patch_size_t,
|
| 394 |
+
height * self.patch_size,
|
| 395 |
+
width * self.patch_size,
|
| 396 |
+
)
|
| 397 |
+
|
| 398 |
+
|
| 399 |
+
class MiniMaxH3VideoVAE(nn.Module):
|
| 400 |
+
def __init__(
|
| 401 |
+
self,
|
| 402 |
+
in_channels=3,
|
| 403 |
+
out_ch=3,
|
| 404 |
+
ch=128,
|
| 405 |
+
embed_dim=24,
|
| 406 |
+
z_channels=24,
|
| 407 |
+
ch_mult=(1, 2, 2, 4, 4, 8),
|
| 408 |
+
num_res_blocks=2,
|
| 409 |
+
space_down=(2, 2, 2, 2, 1, 1),
|
| 410 |
+
time_down=(1, 2, 2, 1, 1, 1),
|
| 411 |
+
clip_length=17,
|
| 412 |
+
token_drop=3,
|
| 413 |
+
tile_size=256,
|
| 414 |
+
tile_overlap_min=64,
|
| 415 |
+
tiling=True,
|
| 416 |
+
):
|
| 417 |
+
super().__init__()
|
| 418 |
+
self.vae_ratio = math.prod(space_down)
|
| 419 |
+
self.vae_ratio_t = math.prod(time_down)
|
| 420 |
+
self.clip_length = clip_length
|
| 421 |
+
self.token_drop = token_drop
|
| 422 |
+
self.frame_pre_padding = (-clip_length) % self.vae_ratio_t
|
| 423 |
+
self.tokens_chunk_size = math.ceil(clip_length / self.vae_ratio_t)
|
| 424 |
+
self.token_overlap = (-token_drop) % self.tokens_chunk_size
|
| 425 |
+
self.frame_overlap = max(self.token_overlap * self.vae_ratio_t - self.frame_pre_padding, 0)
|
| 426 |
+
self.use_tiling = tiling
|
| 427 |
+
self.tile_size = tile_size
|
| 428 |
+
self.tile_overlap_min = tile_overlap_min
|
| 429 |
+
|
| 430 |
+
self.encoder = EncoderFCN3D(
|
| 431 |
+
ch=ch,
|
| 432 |
+
ch_mult=list(ch_mult),
|
| 433 |
+
space_down=list(space_down),
|
| 434 |
+
time_down=list(time_down),
|
| 435 |
+
num_res_blocks=num_res_blocks,
|
| 436 |
+
in_channels=in_channels,
|
| 437 |
+
z_channels=z_channels,
|
| 438 |
+
double_z=True,
|
| 439 |
+
)
|
| 440 |
+
self.quant_conv = nn.Conv3d(2 * z_channels, 2 * embed_dim, kernel_size=1)
|
| 441 |
+
self.post_quant_conv = nn.Conv3d(embed_dim, z_channels, kernel_size=1)
|
| 442 |
+
self.decoder = ViT3DDecoder(
|
| 443 |
+
patch_size=self.vae_ratio,
|
| 444 |
+
patch_size_t=self.vae_ratio_t,
|
| 445 |
+
in_channels=z_channels,
|
| 446 |
+
out_channels=out_ch,
|
| 447 |
+
)
|
| 448 |
+
self.register_buffer("latents_mean", torch.tensor(LATENTS_MEAN))
|
| 449 |
+
self.register_buffer("latents_std", torch.tensor(LATENTS_STD))
|
| 450 |
+
self.register_buffer("pixel_mean", torch.tensor(IMAGENET_MEAN).view(1, 3, 1, 1, 1), persistent=False)
|
| 451 |
+
self.register_buffer("pixel_std", torch.tensor(IMAGENET_STD).view(1, 3, 1, 1, 1), persistent=False)
|
| 452 |
+
|
| 453 |
+
def drop_decoder(self) -> None:
|
| 454 |
+
"""Remove the ViT decoder for encode-only use; `decode` then raises until the VAE is reloaded."""
|
| 455 |
+
self.decoder = None
|
| 456 |
+
self.post_quant_conv = None
|
| 457 |
+
|
| 458 |
+
def _split_tiles(self, length: int) -> tuple[list[int], list[int], list[int]]:
|
| 459 |
+
if self.tile_size >= length:
|
| 460 |
+
return [0], [length], []
|
| 461 |
+
num_tiles = math.ceil(length / self.tile_size)
|
| 462 |
+
while self.tile_size * num_tiles - self.tile_overlap_min * (num_tiles - 1) < length:
|
| 463 |
+
num_tiles += 1
|
| 464 |
+
overlaps = [self.tile_overlap_min] * (num_tiles - 1)
|
| 465 |
+
remaining = self.tile_size * num_tiles - sum(overlaps) - length
|
| 466 |
+
for index in range(remaining // self.vae_ratio):
|
| 467 |
+
overlaps[index % (num_tiles - 1)] += self.vae_ratio
|
| 468 |
+
starts = [0]
|
| 469 |
+
for index in range(num_tiles - 1):
|
| 470 |
+
starts.append(starts[-1] + self.tile_size - overlaps[index])
|
| 471 |
+
return starts, [self.tile_size] * num_tiles, overlaps
|
| 472 |
+
|
| 473 |
+
@staticmethod
|
| 474 |
+
def _blend(first: torch.Tensor, second: torch.Tensor, extent: int, dim: int) -> torch.Tensor:
|
| 475 |
+
extent = min(first.shape[dim], second.shape[dim], extent)
|
| 476 |
+
positions = torch.arange(extent, device=second.device, dtype=second.dtype)
|
| 477 |
+
shape = [1] * first.ndim
|
| 478 |
+
shape[dim] = extent
|
| 479 |
+
first_weight = (1 - positions / extent).view(shape)
|
| 480 |
+
second_weight = (positions / extent).view(shape)
|
| 481 |
+
first_slice = [slice(None)] * first.ndim
|
| 482 |
+
first_slice[dim] = slice(-extent, None)
|
| 483 |
+
second_slice = [slice(None)] * second.ndim
|
| 484 |
+
second_slice[dim] = slice(0, extent)
|
| 485 |
+
blended = first[tuple(first_slice)] * first_weight + second[tuple(second_slice)] * second_weight
|
| 486 |
+
if extent == second.shape[dim]:
|
| 487 |
+
return blended
|
| 488 |
+
rest = [slice(None)] * second.ndim
|
| 489 |
+
rest[dim] = slice(extent, None)
|
| 490 |
+
return torch.cat((blended, second[tuple(rest)]), dim=dim)
|
| 491 |
+
|
| 492 |
+
def _stitch_tiles(
|
| 493 |
+
self,
|
| 494 |
+
tiles: list[list[torch.Tensor]],
|
| 495 |
+
height_overlaps: list[int],
|
| 496 |
+
width_overlaps: list[int],
|
| 497 |
+
) -> torch.Tensor:
|
| 498 |
+
stitched_rows = []
|
| 499 |
+
for row_index, row in enumerate(tiles):
|
| 500 |
+
stitched_row = []
|
| 501 |
+
for column_index, tile in enumerate(row):
|
| 502 |
+
if row_index:
|
| 503 |
+
tile = self._blend(tiles[row_index - 1][column_index], tile, height_overlaps[row_index - 1], -2)
|
| 504 |
+
if column_index:
|
| 505 |
+
tile = self._blend(row[column_index - 1], tile, width_overlaps[column_index - 1], -1)
|
| 506 |
+
if row_index < len(tiles) - 1:
|
| 507 |
+
tile = tile[..., : -height_overlaps[row_index], :]
|
| 508 |
+
if column_index < len(row) - 1:
|
| 509 |
+
tile = tile[..., :, : -width_overlaps[column_index]]
|
| 510 |
+
stitched_row.append(tile)
|
| 511 |
+
stitched_rows.append(torch.cat(stitched_row, dim=-1))
|
| 512 |
+
return torch.cat(stitched_rows, dim=-2)
|
| 513 |
+
|
| 514 |
+
def _encode_clip(self, pixels: torch.Tensor) -> torch.Tensor:
|
| 515 |
+
if not self.use_tiling:
|
| 516 |
+
return self.quant_conv(self.encoder(pixels))
|
| 517 |
+
height_starts, height_lengths, height_overlaps = self._split_tiles(pixels.shape[-2])
|
| 518 |
+
width_starts, width_lengths, width_overlaps = self._split_tiles(pixels.shape[-1])
|
| 519 |
+
tiles = [
|
| 520 |
+
[
|
| 521 |
+
self.quant_conv(
|
| 522 |
+
self.encoder(
|
| 523 |
+
pixels[
|
| 524 |
+
...,
|
| 525 |
+
top : top + tile_height,
|
| 526 |
+
left : left + tile_width,
|
| 527 |
+
]
|
| 528 |
+
)
|
| 529 |
+
)
|
| 530 |
+
for left, tile_width in zip(width_starts, width_lengths)
|
| 531 |
+
]
|
| 532 |
+
for top, tile_height in zip(height_starts, height_lengths)
|
| 533 |
+
]
|
| 534 |
+
latent_height_overlaps = [value // self.vae_ratio for value in height_overlaps]
|
| 535 |
+
latent_width_overlaps = [value // self.vae_ratio for value in width_overlaps]
|
| 536 |
+
return self._stitch_tiles(tiles, latent_height_overlaps, latent_width_overlaps)
|
| 537 |
+
|
| 538 |
+
def _decode_clip(self, latents: torch.Tensor) -> torch.Tensor:
|
| 539 |
+
if not self.use_tiling:
|
| 540 |
+
return self.decoder(self.post_quant_conv(latents))
|
| 541 |
+
height_starts, height_lengths, height_overlaps = self._split_tiles(latents.shape[-2] * self.vae_ratio)
|
| 542 |
+
width_starts, width_lengths, width_overlaps = self._split_tiles(latents.shape[-1] * self.vae_ratio)
|
| 543 |
+
tiles = [
|
| 544 |
+
[
|
| 545 |
+
self.decoder(
|
| 546 |
+
self.post_quant_conv(
|
| 547 |
+
latents[
|
| 548 |
+
...,
|
| 549 |
+
top // self.vae_ratio : (top + tile_height) // self.vae_ratio,
|
| 550 |
+
left // self.vae_ratio : (left + tile_width) // self.vae_ratio,
|
| 551 |
+
]
|
| 552 |
+
)
|
| 553 |
+
)
|
| 554 |
+
for left, tile_width in zip(width_starts, width_lengths)
|
| 555 |
+
]
|
| 556 |
+
for top, tile_height in zip(height_starts, height_lengths)
|
| 557 |
+
]
|
| 558 |
+
return self._stitch_tiles(tiles, height_overlaps, width_overlaps)
|
| 559 |
+
|
| 560 |
+
def _encode_video(self, pixels: torch.Tensor) -> torch.Tensor:
|
| 561 |
+
frame_count = pixels.shape[2]
|
| 562 |
+
if frame_count % self.clip_length:
|
| 563 |
+
padding = pixels[:, :, -1:].repeat(1, 1, (-frame_count) % self.clip_length, 1, 1)
|
| 564 |
+
pixels = torch.cat((pixels, padding), dim=2)
|
| 565 |
+
moments = torch.cat(
|
| 566 |
+
[
|
| 567 |
+
self._encode_clip(pixels[:, :, start : start + self.clip_length])
|
| 568 |
+
for start in range(0, pixels.shape[2], self.clip_length)
|
| 569 |
+
],
|
| 570 |
+
dim=2,
|
| 571 |
+
)
|
| 572 |
+
return moments[:, :, : -self.token_drop] if self.token_drop else moments
|
| 573 |
+
|
| 574 |
+
def _decode_video(self, latents: torch.Tensor) -> torch.Tensor:
|
| 575 |
+
if self.decoder is None:
|
| 576 |
+
raise RuntimeError(
|
| 577 |
+
"MiniMax-H3 video VAE was loaded without its decoder (encode-only); reload it with load_decoder=True"
|
| 578 |
+
)
|
| 579 |
+
chunk_tokens = self.tokens_chunk_size
|
| 580 |
+
chunk_frames = chunk_tokens * self.vae_ratio_t
|
| 581 |
+
padded_token_count = latents.shape[2] + self.token_drop
|
| 582 |
+
pad_tokens = (-padded_token_count) % chunk_tokens
|
| 583 |
+
num_chunks = (padded_token_count + pad_tokens) // chunk_tokens - int(self.token_drop > 0)
|
| 584 |
+
if num_chunks < 1:
|
| 585 |
+
pad_tokens += chunk_tokens
|
| 586 |
+
num_chunks = 1
|
| 587 |
+
if pad_tokens:
|
| 588 |
+
latents = torch.cat((latents, latents[:, :, -1:].repeat(1, 1, pad_tokens, 1, 1)), dim=2)
|
| 589 |
+
|
| 590 |
+
decoded = []
|
| 591 |
+
overlap = None
|
| 592 |
+
for index in range(num_chunks):
|
| 593 |
+
start = index * chunk_tokens
|
| 594 |
+
clip = self._decode_clip(latents[:, :, start : start + chunk_tokens + self.token_overlap])
|
| 595 |
+
for part_index in range(int(self.token_drop > 0) + 1):
|
| 596 |
+
frame_start = part_index * chunk_frames
|
| 597 |
+
part = clip[:, :, frame_start : frame_start + chunk_frames]
|
| 598 |
+
part = part[:, :, self.frame_pre_padding :]
|
| 599 |
+
if part_index == 0:
|
| 600 |
+
if overlap is not None:
|
| 601 |
+
part = self._blend(overlap, part, self.frame_overlap, dim=-3)
|
| 602 |
+
decoded.append(part)
|
| 603 |
+
else:
|
| 604 |
+
overlap = part
|
| 605 |
+
if overlap is not None:
|
| 606 |
+
decoded.append(overlap)
|
| 607 |
+
pixels = torch.cat(decoded, dim=2)
|
| 608 |
+
|
| 609 |
+
if pad_tokens:
|
| 610 |
+
intra_tail = self.clip_length % self.vae_ratio_t
|
| 611 |
+
tokens_before_padding = latents.shape[2] - pad_tokens
|
| 612 |
+
pad_frames = sum(
|
| 613 |
+
intra_tail if intra_tail and (tokens_before_padding + offset) % chunk_tokens == 0 else self.vae_ratio_t
|
| 614 |
+
for offset in range(pad_tokens)
|
| 615 |
+
)
|
| 616 |
+
pixels = pixels[:, :, :-pad_frames]
|
| 617 |
+
return pixels
|
| 618 |
+
|
| 619 |
+
def encode_moments(self, pixels: torch.Tensor) -> torch.Tensor:
|
| 620 |
+
if pixels.ndim == 4:
|
| 621 |
+
pixels = pixels.unsqueeze(2)
|
| 622 |
+
pixels = (pixels + 1.0) * 0.5
|
| 623 |
+
pixels = (pixels - self.pixel_mean.to(pixels)) / self.pixel_std.to(pixels)
|
| 624 |
+
if pixels.shape[2] == 1:
|
| 625 |
+
moments = self._encode_clip(pixels)[:, :, -1:]
|
| 626 |
+
else:
|
| 627 |
+
moments = self._encode_video(pixels)
|
| 628 |
+
return moments.float()
|
| 629 |
+
|
| 630 |
+
def encode(self, pixels: torch.Tensor) -> torch.Tensor:
|
| 631 |
+
mean = self.encode_moments(pixels).chunk(2, dim=1)[0]
|
| 632 |
+
latent_mean = self.latents_mean.view(1, -1, 1, 1, 1).to(mean)
|
| 633 |
+
latent_std = self.latents_std.view(1, -1, 1, 1, 1).to(mean)
|
| 634 |
+
return (mean - latent_mean) / latent_std
|
| 635 |
+
|
| 636 |
+
def encode_tiled(self, pixels: torch.Tensor, **kwargs) -> torch.Tensor:
|
| 637 |
+
del kwargs
|
| 638 |
+
return self.encode(pixels)
|
| 639 |
+
|
| 640 |
+
def decode_tiled(self, latents: torch.Tensor, **kwargs) -> torch.Tensor:
|
| 641 |
+
del kwargs
|
| 642 |
+
return self.decode(latents)
|
| 643 |
+
|
| 644 |
+
def decode(self, latents: torch.Tensor) -> torch.Tensor:
|
| 645 |
+
latent_mean = self.latents_mean.view(1, -1, 1, 1, 1).to(latents)
|
| 646 |
+
latent_std = self.latents_std.view(1, -1, 1, 1, 1).to(latents)
|
| 647 |
+
latents = latents * latent_std + latent_mean
|
| 648 |
+
if latents.shape[2] == 1:
|
| 649 |
+
# The temporal decoder needs at least two tokens of context; a solo token decodes
|
| 650 |
+
# with severe color/scanline artifacts (~15 dB PSNR). Duplicating the token to a
|
| 651 |
+
# pseudo two-token clip and keeping pixel frame 0 stays within ~1-2 dB of a true
|
| 652 |
+
# two-token decode.
|
| 653 |
+
pixels = self._decode_video(latents.repeat(1, 1, 2, 1, 1))[:, :, :1]
|
| 654 |
+
else:
|
| 655 |
+
pixels = self._decode_video(latents)
|
| 656 |
+
pixels = pixels.float() * self.pixel_std.to(pixels) + self.pixel_mean.to(pixels)
|
| 657 |
+
return pixels.clamp(0.0, 1.0) * 2.0 - 1.0
|
| 658 |
+
|
| 659 |
+
|
| 660 |
+
def _video_posterior_sample(vae, pixels: torch.Tensor, generator: torch.Generator, fp16_roundtrip: bool) -> torch.Tensor:
|
| 661 |
+
moments = vae.encode_moments(pixels)
|
| 662 |
+
if moments.ndim != 5 or moments.shape[1] != 48:
|
| 663 |
+
raise ValueError(f"Expected MiniMax-H3 video moments [B,48,T,H,W], got {tuple(moments.shape)}")
|
| 664 |
+
mean, logvar = moments.float().chunk(2, dim=1)
|
| 665 |
+
noise = torch.randn(mean.shape, generator=generator, dtype=torch.float32, device="cpu").to(mean.device)
|
| 666 |
+
sample = mean + torch.exp(0.5 * logvar.clamp(-30.0, 20.0)) * noise
|
| 667 |
+
if fp16_roundtrip:
|
| 668 |
+
sample = sample.to(torch.float16).to(torch.float32)
|
| 669 |
+
latents_mean = vae.latents_mean.view(1, -1, 1, 1, 1).to(sample)
|
| 670 |
+
latents_std = vae.latents_std.view(1, -1, 1, 1, 1).to(sample)
|
| 671 |
+
return (sample - latents_mean) / latents_std
|
| 672 |
+
|
| 673 |
+
|
| 674 |
+
@torch.no_grad()
|
| 675 |
+
def encode_video_target(vae, pixels: torch.Tensor, cache_seed: int, canonical_item_key: str) -> torch.Tensor:
|
| 676 |
+
digest = hashlib.sha256(f"{cache_seed}\0{canonical_item_key}".encode()).digest()
|
| 677 |
+
seed = int.from_bytes(digest[:8], "little") % (2**63)
|
| 678 |
+
generator = torch.Generator(device="cpu").manual_seed(seed)
|
| 679 |
+
return _video_posterior_sample(vae, pixels, generator, fp16_roundtrip=False)
|
| 680 |
+
|
| 681 |
+
|
| 682 |
+
@torch.no_grad()
|
| 683 |
+
def encode_video_condition(vae, pixels: torch.Tensor) -> torch.Tensor:
|
| 684 |
+
generator = torch.Generator(device="cpu").manual_seed(42)
|
| 685 |
+
return _video_posterior_sample(vae, pixels, generator, fp16_roundtrip=True)
|
| 686 |
+
|
| 687 |
+
|
| 688 |
+
def load_video_vae(
|
| 689 |
+
path,
|
| 690 |
+
device: str | torch.device = "cpu",
|
| 691 |
+
dtype: torch.dtype = VIDEO_VAE_DECODE_DTYPE,
|
| 692 |
+
disable_numpy_memmap: bool = False,
|
| 693 |
+
*,
|
| 694 |
+
load_decoder: bool = True,
|
| 695 |
+
) -> MiniMaxH3VideoVAE:
|
| 696 |
+
"""Load the video VAE onto `device` in `dtype`.
|
| 697 |
+
|
| 698 |
+
`load_decoder=False` skips the ViT decoder's tensors entirely (they are never read from the
|
| 699 |
+
file) and returns an encode-only VAE; latent caching uses this since the decoder is ~93% of
|
| 700 |
+
the parameters.
|
| 701 |
+
"""
|
| 702 |
+
from accelerate import init_empty_weights
|
| 703 |
+
|
| 704 |
+
from musubi_tuner.minimax_h3.checkpoint import strip_key_prefixes
|
| 705 |
+
from musubi_tuner.utils.device_utils import synchronize_device
|
| 706 |
+
from musubi_tuner.utils.safetensors_utils import MemoryEfficientSafeOpen
|
| 707 |
+
|
| 708 |
+
with init_empty_weights():
|
| 709 |
+
vae = MiniMaxH3VideoVAE()
|
| 710 |
+
if not load_decoder:
|
| 711 |
+
vae.drop_decoder()
|
| 712 |
+
# loading straight to the target device avoids a resident full-model CPU copy
|
| 713 |
+
device = torch.device(device)
|
| 714 |
+
sd = {}
|
| 715 |
+
with MemoryEfficientSafeOpen(str(path), disable_numpy_memmap=disable_numpy_memmap) as f:
|
| 716 |
+
# stripped model key -> key in the file
|
| 717 |
+
file_keys = strip_key_prefixes({key: key for key in f.keys()}, ("first_stage_model.", "video_vae.", "vae."))
|
| 718 |
+
for key, file_key in file_keys.items():
|
| 719 |
+
if not load_decoder and key.startswith(VIDEO_VAE_DECODER_KEY_PREFIXES):
|
| 720 |
+
continue
|
| 721 |
+
tensor = f.get_tensor(file_key, device=device)
|
| 722 |
+
sd[key] = tensor.to(dtype) if tensor.is_floating_point() else tensor
|
| 723 |
+
synchronize_device(device)
|
| 724 |
+
vae.load_state_dict(sd, strict=True, assign=True)
|
| 725 |
+
vae.to(device)
|
| 726 |
+
vae.eval()
|
| 727 |
+
return vae
|
third_party/musubi_tuner/modules/__init__.py
ADDED
|
File without changes
|
third_party/musubi_tuner/modules/attention.py
ADDED
|
@@ -0,0 +1,351 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Unified attention function supporting various implementations
|
| 2 |
+
|
| 3 |
+
from dataclasses import dataclass
|
| 4 |
+
import torch
|
| 5 |
+
from typing import Optional, Union
|
| 6 |
+
|
| 7 |
+
try:
|
| 8 |
+
import flash_attn
|
| 9 |
+
from flash_attn.flash_attn_interface import _flash_attn_forward
|
| 10 |
+
from flash_attn.flash_attn_interface import flash_attn_varlen_func
|
| 11 |
+
from flash_attn.flash_attn_interface import flash_attn_func
|
| 12 |
+
except ImportError:
|
| 13 |
+
flash_attn = None
|
| 14 |
+
flash_attn_varlen_func = None
|
| 15 |
+
_flash_attn_forward = None
|
| 16 |
+
flash_attn_func = None
|
| 17 |
+
|
| 18 |
+
try:
|
| 19 |
+
import flash_attn_interface # FlashAttention 3 (Hopper)
|
| 20 |
+
except ImportError:
|
| 21 |
+
flash_attn_interface = None
|
| 22 |
+
|
| 23 |
+
try:
|
| 24 |
+
from sageattention import sageattn_varlen, sageattn
|
| 25 |
+
except ImportError:
|
| 26 |
+
sageattn_varlen = None
|
| 27 |
+
sageattn = None
|
| 28 |
+
|
| 29 |
+
try:
|
| 30 |
+
import xformers.ops as xops
|
| 31 |
+
except ImportError:
|
| 32 |
+
xops = None
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
@dataclass
|
| 36 |
+
class AttentionParams:
|
| 37 |
+
attn_mode: Optional[str] = None
|
| 38 |
+
split_attn: bool = False
|
| 39 |
+
img_len: Optional[int] = None
|
| 40 |
+
attention_mask: Optional[torch.Tensor] = None
|
| 41 |
+
seqlens: Optional[torch.Tensor] = None
|
| 42 |
+
cu_seqlens: Optional[torch.Tensor] = None
|
| 43 |
+
max_seqlen: Optional[int] = None
|
| 44 |
+
|
| 45 |
+
def __post_init__(self):
|
| 46 |
+
if self.attn_mode == "sdpa":
|
| 47 |
+
self.attn_mode = "torch"
|
| 48 |
+
|
| 49 |
+
@staticmethod
|
| 50 |
+
def create_attention_params(attn_mode: Optional[str], split_attn: bool) -> "AttentionParams":
|
| 51 |
+
return AttentionParams(attn_mode, split_attn)
|
| 52 |
+
|
| 53 |
+
@staticmethod
|
| 54 |
+
def create_attention_params_from_mask(
|
| 55 |
+
attn_mode: Optional[str], split_attn: bool, img_len: Optional[int], attention_mask: Optional[torch.Tensor]
|
| 56 |
+
) -> "AttentionParams":
|
| 57 |
+
if attention_mask is None:
|
| 58 |
+
# No attention mask provided: assume all tokens are valid
|
| 59 |
+
return AttentionParams(attn_mode, split_attn, None, None, None, None, None)
|
| 60 |
+
else:
|
| 61 |
+
# Note: attention_mask is only for text tokens, not including image tokens
|
| 62 |
+
seqlens = attention_mask.sum(dim=1).to(torch.int32) + img_len # [B]
|
| 63 |
+
max_seqlen = attention_mask.shape[1] + img_len
|
| 64 |
+
|
| 65 |
+
if split_attn:
|
| 66 |
+
# cu_seqlens is not needed for split attention
|
| 67 |
+
return AttentionParams(attn_mode, split_attn, img_len, attention_mask, seqlens, None, max_seqlen)
|
| 68 |
+
|
| 69 |
+
# Convert attention mask to cumulative sequence lengths for flash attention
|
| 70 |
+
batch_size = attention_mask.shape[0]
|
| 71 |
+
cu_seqlens = torch.zeros([2 * batch_size + 1], dtype=torch.int32, device=attention_mask.device)
|
| 72 |
+
for i in range(batch_size):
|
| 73 |
+
cu_seqlens[2 * i + 1] = i * max_seqlen + seqlens[i] # end of valid tokens for query
|
| 74 |
+
cu_seqlens[2 * i + 2] = (i + 1) * max_seqlen # end of all tokens for query
|
| 75 |
+
|
| 76 |
+
# Expand attention mask to include image tokens
|
| 77 |
+
attention_mask = torch.nn.functional.pad(attention_mask, (img_len, 0), value=1) # [B, img_len + L]
|
| 78 |
+
|
| 79 |
+
# attention bias for xformers
|
| 80 |
+
if attn_mode == "xformers":
|
| 81 |
+
seqlens_list = seqlens.cpu().tolist()
|
| 82 |
+
attention_mask = xops.fmha.attn_bias.BlockDiagonalMask.from_seqlens(
|
| 83 |
+
seqlens_list, seqlens_list, device=attention_mask.device
|
| 84 |
+
)
|
| 85 |
+
elif attn_mode == "torch":
|
| 86 |
+
attention_mask = attention_mask[:, None, None, :].to(torch.bool) # [B, 1, 1, img_len + L]
|
| 87 |
+
|
| 88 |
+
return AttentionParams(attn_mode, split_attn, img_len, attention_mask, seqlens, cu_seqlens, max_seqlen)
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
def _flash3_output(result):
|
| 92 |
+
# older FlashAttention 3 releases return (out, softmax_lse); newer ones return out only
|
| 93 |
+
return result[0] if isinstance(result, tuple) else result
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
def attention(
|
| 97 |
+
qkv_or_q: Union[torch.Tensor, list],
|
| 98 |
+
k: Optional[torch.Tensor] = None,
|
| 99 |
+
v: Optional[torch.Tensor] = None,
|
| 100 |
+
attn_params: Optional[AttentionParams] = None,
|
| 101 |
+
drop_rate: float = 0.0,
|
| 102 |
+
) -> torch.Tensor:
|
| 103 |
+
"""
|
| 104 |
+
Compute scaled dot-product attention with variable sequence lengths.
|
| 105 |
+
|
| 106 |
+
Handles batches with different sequence lengths by splitting and
|
| 107 |
+
processing each sequence individually.
|
| 108 |
+
|
| 109 |
+
Args:
|
| 110 |
+
qkv_or_q: Query tensor [B, L, H, D]. or list of such tensors.
|
| 111 |
+
k: Key tensor [B, L, H, D].
|
| 112 |
+
v: Value tensor [B, L, H, D].
|
| 113 |
+
attn_param: Attention parameters including mask and sequence lengths.
|
| 114 |
+
drop_rate: Attention dropout rate.
|
| 115 |
+
|
| 116 |
+
Returns:
|
| 117 |
+
Attention output tensor [B, L, H*D].
|
| 118 |
+
"""
|
| 119 |
+
if isinstance(qkv_or_q, list):
|
| 120 |
+
q, k, v = qkv_or_q
|
| 121 |
+
q: torch.Tensor = q
|
| 122 |
+
qkv_or_q.clear()
|
| 123 |
+
del qkv_or_q
|
| 124 |
+
else:
|
| 125 |
+
q: torch.Tensor = qkv_or_q
|
| 126 |
+
del qkv_or_q
|
| 127 |
+
assert k is not None and v is not None, "k and v must be provided if qkv_or_q is a tensor"
|
| 128 |
+
if attn_params is None:
|
| 129 |
+
attn_params = AttentionParams.create_attention_params("torch", False)
|
| 130 |
+
|
| 131 |
+
# GQA: q may carry more heads than k/v (e.g. Krea 2 = 48 query / 12 kv heads). flash and
|
| 132 |
+
# sageattn group heads natively inside the kernel (verified), so they ignore this. For the
|
| 133 |
+
# torch (SDPA) path we expand k/v to q's head count below instead of passing enable_gqa=True,
|
| 134 |
+
# because enable_gqa forces SDPA onto the slow math kernel (~7x slower than the fused kernels
|
| 135 |
+
# at K2 scale); the repeat is numerically identical. Revert to enable_gqa=True once PyTorch's
|
| 136 |
+
# fused (flash / mem-efficient) kernels gain native GQA support. (q/k/v here are [B, L, H, D].)
|
| 137 |
+
enable_gqa = q.shape[-2] != k.shape[-2]
|
| 138 |
+
|
| 139 |
+
# If split attn is False, attention mask is provided and all sequence lengths are same, we can trim the sequence
|
| 140 |
+
seqlen_trimmed = False
|
| 141 |
+
# Trim if all seqlens are the same, for attention modes other than flash or sageattn (which can handle masks efficiently)
|
| 142 |
+
if (
|
| 143 |
+
not attn_params.split_attn
|
| 144 |
+
and attn_params.attention_mask is not None
|
| 145 |
+
and attn_params.seqlens is not None
|
| 146 |
+
and (attn_params.attn_mode != "flash" and attn_params.attn_mode != "sageattn")
|
| 147 |
+
):
|
| 148 |
+
if torch.all(attn_params.seqlens == attn_params.seqlens[0]):
|
| 149 |
+
seqlen = attn_params.seqlens[0].item()
|
| 150 |
+
q = q[:, :seqlen]
|
| 151 |
+
k = k[:, :seqlen]
|
| 152 |
+
v = v[:, :seqlen]
|
| 153 |
+
max_seqlen = attn_params.max_seqlen
|
| 154 |
+
attn_params = AttentionParams.create_attention_params(attn_params.attn_mode, False) # do not in-place modify
|
| 155 |
+
attn_params.max_seqlen = max_seqlen # keep max_seqlen for padding
|
| 156 |
+
seqlen_trimmed = True
|
| 157 |
+
|
| 158 |
+
# Determine tensor layout based on attention implementation
|
| 159 |
+
if attn_params.attn_mode == "torch" or (
|
| 160 |
+
attn_params.attn_mode == "sageattn" and (attn_params.split_attn or attn_params.cu_seqlens is None)
|
| 161 |
+
):
|
| 162 |
+
transpose_fn = lambda x: x.transpose(1, 2) # [B, H, L, D] for SDPA and sageattn with fixed length
|
| 163 |
+
# pad on sequence length dimension
|
| 164 |
+
pad_fn = lambda x, pad_to: torch.nn.functional.pad(x, (0, 0, 0, pad_to - x.shape[-2]), value=0)
|
| 165 |
+
else:
|
| 166 |
+
transpose_fn = lambda x: x # [B, L, H, D] for other implementations
|
| 167 |
+
# pad on sequence length dimension
|
| 168 |
+
pad_fn = lambda x, pad_to: torch.nn.functional.pad(x, (0, 0, 0, 0, 0, pad_to - x.shape[-3]), value=0)
|
| 169 |
+
|
| 170 |
+
# Process each batch element with its valid sequence lengths
|
| 171 |
+
if attn_params.split_attn:
|
| 172 |
+
if attn_params.seqlens is None:
|
| 173 |
+
# If no seqlens provided, assume all tokens are valid
|
| 174 |
+
attn_params = AttentionParams.create_attention_params(attn_params.attn_mode, True) # do not in-place modify
|
| 175 |
+
attn_params.seqlens = torch.tensor([q.shape[1]] * q.shape[0], device=q.device)
|
| 176 |
+
attn_params.max_seqlen = q.shape[1]
|
| 177 |
+
q = [transpose_fn(q[i : i + 1, : attn_params.seqlens[i]]) for i in range(len(q))]
|
| 178 |
+
k = [transpose_fn(k[i : i + 1, : attn_params.seqlens[i]]) for i in range(len(k))]
|
| 179 |
+
v = [transpose_fn(v[i : i + 1, : attn_params.seqlens[i]]) for i in range(len(v))]
|
| 180 |
+
else:
|
| 181 |
+
q = transpose_fn(q)
|
| 182 |
+
k = transpose_fn(k)
|
| 183 |
+
v = transpose_fn(v)
|
| 184 |
+
|
| 185 |
+
if attn_params.attn_mode == "torch":
|
| 186 |
+
if attn_params.split_attn:
|
| 187 |
+
x = []
|
| 188 |
+
for i in range(len(q)):
|
| 189 |
+
qi, ki, vi = q[i], k[i], v[i]
|
| 190 |
+
if enable_gqa: # expand k/v heads to avoid SDPA's slow enable_gqa math path
|
| 191 |
+
g = qi.shape[1] // ki.shape[1] # [B, H, L, D] -> heads at dim 1
|
| 192 |
+
ki = ki.repeat_interleave(g, dim=1)
|
| 193 |
+
vi = vi.repeat_interleave(g, dim=1)
|
| 194 |
+
x_i = torch.nn.functional.scaled_dot_product_attention(qi, ki, vi, dropout_p=drop_rate)
|
| 195 |
+
q[i] = None
|
| 196 |
+
k[i] = None
|
| 197 |
+
v[i] = None
|
| 198 |
+
x.append(pad_fn(x_i, attn_params.max_seqlen)) # B, H, L, D
|
| 199 |
+
x = torch.cat(x, dim=0)
|
| 200 |
+
q, k, v = None, None, None
|
| 201 |
+
|
| 202 |
+
else:
|
| 203 |
+
if enable_gqa: # expand k/v heads to avoid SDPA's slow enable_gqa math path
|
| 204 |
+
g = q.shape[1] // k.shape[1] # [B, H, L, D] -> heads at dim 1
|
| 205 |
+
k = k.repeat_interleave(g, dim=1)
|
| 206 |
+
v = v.repeat_interleave(g, dim=1)
|
| 207 |
+
x = torch.nn.functional.scaled_dot_product_attention(q, k, v, attn_mask=attn_params.attention_mask, dropout_p=drop_rate)
|
| 208 |
+
q, k, v = None, None, None
|
| 209 |
+
|
| 210 |
+
elif attn_params.attn_mode == "xformers":
|
| 211 |
+
if attn_params.split_attn:
|
| 212 |
+
x = []
|
| 213 |
+
for i in range(len(q)):
|
| 214 |
+
qi, ki, vi = q[i], k[i], v[i]
|
| 215 |
+
if enable_gqa: # expand k/v heads to q's count; xformers 4D has no native GQA ([B, L, H, D] -> dim 2)
|
| 216 |
+
g = qi.shape[2] // ki.shape[2]
|
| 217 |
+
ki = ki.repeat_interleave(g, dim=2)
|
| 218 |
+
vi = vi.repeat_interleave(g, dim=2)
|
| 219 |
+
x_i = xops.memory_efficient_attention(qi, ki, vi, p=drop_rate)
|
| 220 |
+
q[i] = None
|
| 221 |
+
k[i] = None
|
| 222 |
+
v[i] = None
|
| 223 |
+
x.append(pad_fn(x_i, attn_params.max_seqlen)) # B, L, H, D
|
| 224 |
+
x = torch.cat(x, dim=0)
|
| 225 |
+
q, k, v = None, None, None
|
| 226 |
+
|
| 227 |
+
else:
|
| 228 |
+
if enable_gqa: # expand k/v heads to q's count; xformers 4D has no native GQA ([B, L, H, D] -> dim 2)
|
| 229 |
+
g = q.shape[2] // k.shape[2]
|
| 230 |
+
k = k.repeat_interleave(g, dim=2)
|
| 231 |
+
v = v.repeat_interleave(g, dim=2)
|
| 232 |
+
x = xops.memory_efficient_attention(q, k, v, attn_bias=attn_params.attention_mask, p=drop_rate)
|
| 233 |
+
q, k, v = None, None, None
|
| 234 |
+
|
| 235 |
+
elif attn_params.attn_mode == "sageattn":
|
| 236 |
+
if attn_params.split_attn:
|
| 237 |
+
x = []
|
| 238 |
+
for i in range(len(q)):
|
| 239 |
+
# HND seems to cause an error
|
| 240 |
+
x_i = sageattn(q[i], k[i], v[i]) # B, H, L, D. No dropout support
|
| 241 |
+
q[i] = None
|
| 242 |
+
k[i] = None
|
| 243 |
+
v[i] = None
|
| 244 |
+
x.append(pad_fn(x_i, attn_params.max_seqlen)) # B, H, L, D
|
| 245 |
+
x = torch.cat(x, dim=0)
|
| 246 |
+
q, k, v = None, None, None
|
| 247 |
+
elif attn_params.cu_seqlens is None: # all tokens are valid
|
| 248 |
+
x = sageattn(q, k, v) # B, L, H, D. No dropout support
|
| 249 |
+
q, k, v = None, None, None
|
| 250 |
+
else:
|
| 251 |
+
# Reshape to [(bxs), a, d]
|
| 252 |
+
batch_size, seqlen = q.shape[0], q.shape[1]
|
| 253 |
+
# reshape (not view): callers may pass non-contiguous q/k/v (e.g. Krea 2 transposes
|
| 254 |
+
# [B,H,L,D]->[B,L,H,D] after RoPE); reshape copies only when needed, view-equivalent otherwise.
|
| 255 |
+
q = q.reshape(q.shape[0] * q.shape[1], *q.shape[2:]) # [B*L, H, D]
|
| 256 |
+
k = k.reshape(k.shape[0] * k.shape[1], *k.shape[2:]) # [B*L, H, D]
|
| 257 |
+
v = v.reshape(v.shape[0] * v.shape[1], *v.shape[2:]) # [B*L, H, D]
|
| 258 |
+
|
| 259 |
+
# Assume cu_seqlens_q == cu_seqlens_kv and max_seqlen_q == max_seqlen_kv. No dropout support
|
| 260 |
+
x = sageattn_varlen(
|
| 261 |
+
q, k, v, attn_params.cu_seqlens, attn_params.cu_seqlens, attn_params.max_seqlen, attn_params.max_seqlen
|
| 262 |
+
)
|
| 263 |
+
q, k, v = None, None, None
|
| 264 |
+
|
| 265 |
+
# Reshape x with shape [(bxs), a, d] to [b, s, a, d]
|
| 266 |
+
x = x.view(batch_size, seqlen, x.shape[-2], x.shape[-1]) # B, L, H, D
|
| 267 |
+
|
| 268 |
+
elif attn_params.attn_mode == "flash":
|
| 269 |
+
if attn_params.split_attn:
|
| 270 |
+
x = []
|
| 271 |
+
for i in range(len(q)):
|
| 272 |
+
# HND seems to cause an error
|
| 273 |
+
x_i = flash_attn_func(q[i], k[i], v[i], drop_rate) # B, L, H, D
|
| 274 |
+
q[i] = None
|
| 275 |
+
k[i] = None
|
| 276 |
+
v[i] = None
|
| 277 |
+
x.append(pad_fn(x_i, attn_params.max_seqlen)) # B, L, H, D
|
| 278 |
+
x = torch.cat(x, dim=0)
|
| 279 |
+
q, k, v = None, None, None
|
| 280 |
+
elif attn_params.cu_seqlens is None: # all tokens are valid
|
| 281 |
+
x = flash_attn_func(q, k, v, drop_rate) # B, L, H, D
|
| 282 |
+
q, k, v = None, None, None
|
| 283 |
+
else:
|
| 284 |
+
# Reshape to [(bxs), a, d]
|
| 285 |
+
batch_size, seqlen = q.shape[0], q.shape[1]
|
| 286 |
+
# reshape (not view): callers may pass non-contiguous q/k/v (e.g. Krea 2 transposes
|
| 287 |
+
# [B,H,L,D]->[B,L,H,D] after RoPE); reshape copies only when needed, view-equivalent otherwise.
|
| 288 |
+
q = q.reshape(q.shape[0] * q.shape[1], *q.shape[2:]) # [B*L, H, D]
|
| 289 |
+
k = k.reshape(k.shape[0] * k.shape[1], *k.shape[2:]) # [B*L, H, D]
|
| 290 |
+
v = v.reshape(v.shape[0] * v.shape[1], *v.shape[2:]) # [B*L, H, D]
|
| 291 |
+
|
| 292 |
+
# Assume cu_seqlens_q == cu_seqlens_kv and max_seqlen_q == max_seqlen_kv
|
| 293 |
+
x = flash_attn_varlen_func(
|
| 294 |
+
q, k, v, attn_params.cu_seqlens, attn_params.cu_seqlens, attn_params.max_seqlen, attn_params.max_seqlen, drop_rate
|
| 295 |
+
)
|
| 296 |
+
q, k, v = None, None, None
|
| 297 |
+
|
| 298 |
+
# Reshape x with shape [(bxs), a, d] to [b, s, a, d]
|
| 299 |
+
x = x.view(batch_size, seqlen, x.shape[-2], x.shape[-1]) # B, L, H, D
|
| 300 |
+
|
| 301 |
+
elif attn_params.attn_mode == "flash3":
|
| 302 |
+
# FlashAttention 3 groups GQA heads in the kernel like flash; it has no dropout support
|
| 303 |
+
if flash_attn_interface is None:
|
| 304 |
+
raise RuntimeError("FlashAttention 3 was selected but flash_attn_interface is not installed")
|
| 305 |
+
if attn_params.split_attn:
|
| 306 |
+
x = []
|
| 307 |
+
for i in range(len(q)):
|
| 308 |
+
x_i = _flash3_output(flash_attn_interface.flash_attn_func(q[i], k[i], v[i])) # B, L, H, D
|
| 309 |
+
q[i] = None
|
| 310 |
+
k[i] = None
|
| 311 |
+
v[i] = None
|
| 312 |
+
x.append(pad_fn(x_i, attn_params.max_seqlen)) # B, L, H, D
|
| 313 |
+
x = torch.cat(x, dim=0)
|
| 314 |
+
q, k, v = None, None, None
|
| 315 |
+
elif attn_params.cu_seqlens is None: # all tokens are valid
|
| 316 |
+
x = _flash3_output(flash_attn_interface.flash_attn_func(q, k, v)) # B, L, H, D
|
| 317 |
+
q, k, v = None, None, None
|
| 318 |
+
else:
|
| 319 |
+
# Reshape to [(bxs), a, d]
|
| 320 |
+
batch_size, seqlen = q.shape[0], q.shape[1]
|
| 321 |
+
q = q.reshape(q.shape[0] * q.shape[1], *q.shape[2:]) # [B*L, H, D]
|
| 322 |
+
k = k.reshape(k.shape[0] * k.shape[1], *k.shape[2:]) # [B*L, H, D]
|
| 323 |
+
v = v.reshape(v.shape[0] * v.shape[1], *v.shape[2:]) # [B*L, H, D]
|
| 324 |
+
|
| 325 |
+
# Assume cu_seqlens_q == cu_seqlens_kv and max_seqlen_q == max_seqlen_kv
|
| 326 |
+
x = _flash3_output(
|
| 327 |
+
flash_attn_interface.flash_attn_varlen_func(
|
| 328 |
+
q,
|
| 329 |
+
k,
|
| 330 |
+
v,
|
| 331 |
+
cu_seqlens_q=attn_params.cu_seqlens,
|
| 332 |
+
cu_seqlens_k=attn_params.cu_seqlens,
|
| 333 |
+
max_seqlen_q=attn_params.max_seqlen,
|
| 334 |
+
max_seqlen_k=attn_params.max_seqlen,
|
| 335 |
+
)
|
| 336 |
+
)
|
| 337 |
+
q, k, v = None, None, None
|
| 338 |
+
|
| 339 |
+
# Reshape x with shape [(bxs), a, d] to [b, s, a, d]
|
| 340 |
+
x = x.view(batch_size, seqlen, x.shape[-2], x.shape[-1]) # B, L, H, D
|
| 341 |
+
|
| 342 |
+
else:
|
| 343 |
+
raise ValueError(f"Unsupported attention mode: {attn_params.attn_mode}")
|
| 344 |
+
|
| 345 |
+
x = transpose_fn(x) # [B, L, H, D]
|
| 346 |
+
x = x.reshape(x.shape[0], x.shape[1], -1) # [B, L, H*D]
|
| 347 |
+
|
| 348 |
+
if seqlen_trimmed:
|
| 349 |
+
x = torch.nn.functional.pad(x, (0, 0, 0, attn_params.max_seqlen - x.shape[1]), value=0) # pad back to max_seqlen
|
| 350 |
+
|
| 351 |
+
return x
|
third_party/musubi_tuner/modules/comfy_quant_utils.py
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Shared helpers for ComfyUI ``.comfy_quant`` pre-quantized checkpoint specs.
|
| 2 |
+
|
| 3 |
+
Each quantized module in a ComfyUI checkpoint carries a tiny sibling tensor
|
| 4 |
+
``<module>.comfy_quant`` holding the uint8 bytes of a JSON object that declares the
|
| 5 |
+
module's quantization format. This module only decodes the JSON and classifies
|
| 6 |
+
formats so callers can route a checkpoint to the right loader; format-specific
|
| 7 |
+
validation and conversion live with the loaders (``convrot_int8_utils`` for ConvRot
|
| 8 |
+
INT8, ``nvfp4_utils`` for NVFP4 + per-row INT8 embeddings).
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
import json
|
| 12 |
+
from collections.abc import Iterable
|
| 13 |
+
from pathlib import Path
|
| 14 |
+
from typing import Set, Union
|
| 15 |
+
|
| 16 |
+
import torch
|
| 17 |
+
|
| 18 |
+
from musubi_tuner.utils.safetensors_utils import MemoryEfficientSafeOpen
|
| 19 |
+
|
| 20 |
+
COMFY_QUANT_SUFFIX = ".comfy_quant"
|
| 21 |
+
COMFY_WEIGHT_SCALE_SUFFIX = ".weight_scale"
|
| 22 |
+
|
| 23 |
+
# canonical labels returned by classify_comfy_quant_spec / detect_comfy_quant_formats
|
| 24 |
+
FORMAT_CONVROT_INT8 = "convrot_int8"
|
| 25 |
+
FORMAT_INT8_TENSORWISE = "int8_tensorwise"
|
| 26 |
+
FORMAT_NVFP4 = "nvfp4"
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def decode_comfy_quant_spec(key: str, tensor: torch.Tensor) -> dict:
|
| 30 |
+
"""Decode a ``.comfy_quant`` tensor (uint8 bytes of JSON) into a dict, format-agnostic."""
|
| 31 |
+
if tensor.dtype != torch.uint8 or tensor.ndim != 1:
|
| 32 |
+
raise ValueError(f"Invalid comfy_quant tensor for {key}: expected 1D uint8, got {tensor.dtype} ndim={tensor.ndim}")
|
| 33 |
+
try:
|
| 34 |
+
spec = json.loads(bytes(tensor.tolist()).decode("utf-8"))
|
| 35 |
+
except (UnicodeDecodeError, json.JSONDecodeError) as e:
|
| 36 |
+
raise ValueError(f"Invalid comfy_quant JSON for {key}: {e}") from e
|
| 37 |
+
if not isinstance(spec, dict):
|
| 38 |
+
raise ValueError(f"Invalid comfy_quant spec for {key}: expected a JSON object, got {type(spec).__name__}")
|
| 39 |
+
return spec
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def classify_comfy_quant_spec(spec: dict) -> str:
|
| 43 |
+
"""Map a decoded spec to a canonical format label (unknown formats pass through as-is)."""
|
| 44 |
+
quant_format = spec.get("format")
|
| 45 |
+
if quant_format == FORMAT_INT8_TENSORWISE and spec.get("convrot"):
|
| 46 |
+
return FORMAT_CONVROT_INT8
|
| 47 |
+
return str(quant_format)
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def detect_comfy_quant_formats(files: Iterable[Union[str, Path]], *, disable_numpy_memmap: bool = False) -> Set[str]:
|
| 51 |
+
"""Collect the canonical format labels a checkpoint declares (empty set: not pre-quantized).
|
| 52 |
+
|
| 53 |
+
Routing only — the spec tensors are tiny, so reading them here is cheap; per-format
|
| 54 |
+
validation happens again inside the selected loader.
|
| 55 |
+
"""
|
| 56 |
+
formats: Set[str] = set()
|
| 57 |
+
for file in files:
|
| 58 |
+
with MemoryEfficientSafeOpen(str(file), disable_numpy_memmap=disable_numpy_memmap) as f:
|
| 59 |
+
for key in f.keys():
|
| 60 |
+
if key.endswith(COMFY_QUANT_SUFFIX):
|
| 61 |
+
formats.add(classify_comfy_quant_spec(decode_comfy_quant_spec(key, f.get_tensor(key))))
|
| 62 |
+
return formats
|
third_party/musubi_tuner/modules/convrot_int8_kernels.py
ADDED
|
@@ -0,0 +1,546 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
| 2 |
+
# SPDX-FileCopyrightText: Copyright (c) 2025 Comfy Org
|
| 3 |
+
# SPDX-License-Identifier: Apache-2.0
|
| 4 |
+
#
|
| 5 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 6 |
+
# you may not use this file except in compliance with the License.
|
| 7 |
+
# You may obtain a copy of the License at
|
| 8 |
+
#
|
| 9 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 10 |
+
#
|
| 11 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 12 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 13 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 14 |
+
# See the License for the specific language governing permissions and
|
| 15 |
+
# limitations under the License.
|
| 16 |
+
#
|
| 17 |
+
# Vendored from comfy-kitchen (https://github.com/Comfy-Org/comfy-kitchen):
|
| 18 |
+
# - backends/triton/quantization.py (INT8 section, itself derived from dxqb/OneTrainer
|
| 19 |
+
# and ComfyUI-Flux2-INT8)
|
| 20 |
+
# - tensor/int8_utils.py (Hadamard rotation helpers)
|
| 21 |
+
# - backends/eager/quantization.py (offline row-wise INT8 quantization)
|
| 22 |
+
# Modifications for Musubi Tuner: triton made an optional import (eager fallback when
|
| 23 |
+
# unavailable), stochastic rounding removed, unused formats (fp8/nvfp4/mxfp8) dropped,
|
| 24 |
+
# helpers inlined to make the file self-contained.
|
| 25 |
+
|
| 26 |
+
"""ConvRot INT8 kernels: block-diagonal Hadamard rotation + row-wise INT8 quantization.
|
| 27 |
+
|
| 28 |
+
ConvRot (arXiv:2512.03673) smooths activation outliers by rotating each group of
|
| 29 |
+
`group_size` input features with a normalized regular Hadamard matrix H (symmetric,
|
| 30 |
+
orthogonal: H @ H = I) before quantization:
|
| 31 |
+
|
| 32 |
+
- offline: W_rot = W @ H^T (per group), then per-output-channel INT8 quantization
|
| 33 |
+
- online: x_rot = x @ H (per group), dynamic per-row INT8 quantization, INT8 GEMM
|
| 34 |
+
with the dequantization epilogue fused into the matmul kernel
|
| 35 |
+
|
| 36 |
+
Since H is symmetric orthogonal, y = x_rot @ W_rot^T equals x @ W^T up to
|
| 37 |
+
quantization error.
|
| 38 |
+
"""
|
| 39 |
+
|
| 40 |
+
import math
|
| 41 |
+
|
| 42 |
+
import torch
|
| 43 |
+
|
| 44 |
+
try:
|
| 45 |
+
import triton
|
| 46 |
+
import triton.language as tl
|
| 47 |
+
from triton.language.extra import libdevice
|
| 48 |
+
|
| 49 |
+
HAS_TRITON = True
|
| 50 |
+
except ImportError:
|
| 51 |
+
HAS_TRITON = False
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
# =============================================================================
|
| 55 |
+
# Hadamard rotation helpers (from comfy_kitchen/tensor/int8_utils.py)
|
| 56 |
+
# =============================================================================
|
| 57 |
+
|
| 58 |
+
_HADAMARD_CACHE = {}
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def _build_hadamard(
|
| 62 |
+
size: int,
|
| 63 |
+
device: str | torch.device = "cpu",
|
| 64 |
+
dtype: torch.dtype = torch.float32,
|
| 65 |
+
) -> torch.Tensor:
|
| 66 |
+
"""Build a normalized REGULAR orthogonal Hadamard matrix (ConvRot)."""
|
| 67 |
+
cache_key = (size, str(device), dtype)
|
| 68 |
+
if cache_key in _HADAMARD_CACHE:
|
| 69 |
+
return _HADAMARD_CACHE[cache_key]
|
| 70 |
+
|
| 71 |
+
if size < 4 or (size & (size - 1)) != 0 or math.log(size, 4) % 1 != 0:
|
| 72 |
+
raise ValueError(f"Regular Hadamard size must be a power of 4, got {size}")
|
| 73 |
+
|
| 74 |
+
h4 = torch.tensor(
|
| 75 |
+
[[1, 1, 1, -1], [1, 1, -1, 1], [1, -1, 1, 1], [-1, 1, 1, 1]],
|
| 76 |
+
dtype=dtype,
|
| 77 |
+
device=device,
|
| 78 |
+
)
|
| 79 |
+
|
| 80 |
+
h = h4
|
| 81 |
+
current_size = 4
|
| 82 |
+
while current_size < size:
|
| 83 |
+
h = torch.kron(h, h4)
|
| 84 |
+
current_size *= 4
|
| 85 |
+
|
| 86 |
+
h_normalized = h / (size**0.5)
|
| 87 |
+
_HADAMARD_CACHE[cache_key] = h_normalized
|
| 88 |
+
return h_normalized
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
def _rotate_weight(
|
| 92 |
+
weight: torch.Tensor,
|
| 93 |
+
h: torch.Tensor,
|
| 94 |
+
group_size: int,
|
| 95 |
+
) -> torch.Tensor:
|
| 96 |
+
"""Rotate weight matrix offline: W_rot = W @ H_block^T."""
|
| 97 |
+
out_f, in_f = weight.shape
|
| 98 |
+
if in_f % group_size != 0:
|
| 99 |
+
raise ValueError(f"in_features {in_f} not divisible by group_size {group_size}")
|
| 100 |
+
n_groups = in_f // group_size
|
| 101 |
+
|
| 102 |
+
weight_grouped = weight.reshape(out_f, n_groups, group_size)
|
| 103 |
+
h_t = h.T.to(dtype=weight.dtype, device=weight.device)
|
| 104 |
+
weight_rotated = torch.matmul(weight_grouped, h_t)
|
| 105 |
+
return weight_rotated.reshape(out_f, in_f)
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def _rotate_activation(
|
| 109 |
+
x: torch.Tensor,
|
| 110 |
+
h: torch.Tensor,
|
| 111 |
+
group_size: int,
|
| 112 |
+
) -> torch.Tensor:
|
| 113 |
+
"""Rotate activation online using Optimized Matmul implementation."""
|
| 114 |
+
orig_shape = x.shape
|
| 115 |
+
features = orig_shape[-1]
|
| 116 |
+
if features % group_size != 0:
|
| 117 |
+
raise ValueError(f"features {features} not divisible by group_size {group_size}")
|
| 118 |
+
n_groups = features // group_size
|
| 119 |
+
|
| 120 |
+
x_grouped = x.reshape(-1, n_groups, group_size)
|
| 121 |
+
h = h.to(dtype=x.dtype, device=x.device)
|
| 122 |
+
x_rotated = torch.matmul(x_grouped, h)
|
| 123 |
+
|
| 124 |
+
return x_rotated.reshape(orig_shape)
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
# =============================================================================
|
| 128 |
+
# Offline row-wise INT8 quantization (from comfy_kitchen/backends/eager/quantization.py,
|
| 129 |
+
# stochastic rounding removed)
|
| 130 |
+
# =============================================================================
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
def _int8_scale_for_math(scale: torch.Tensor, x: torch.Tensor) -> torch.Tensor:
|
| 134 |
+
scale = scale.to(device=x.device, dtype=x.dtype)
|
| 135 |
+
scale_min = torch.finfo(x.dtype).tiny
|
| 136 |
+
return torch.where(scale == 0, torch.full_like(scale, scale_min), scale)
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
def _round_int8(scaled: torch.Tensor) -> torch.Tensor:
|
| 140 |
+
return scaled.round_().clamp_(-128.0, 127.0).to(torch.int8)
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
def quantize_int8_rowwise(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
| 144 |
+
"""Quantize tensor to INT8 with per-row scales.
|
| 145 |
+
|
| 146 |
+
Returns:
|
| 147 |
+
Tuple of (quantized_int8 [..., K], scales float32 [..., 1]).
|
| 148 |
+
"""
|
| 149 |
+
abs_max = x.abs().amax(dim=-1, keepdim=True)
|
| 150 |
+
scale = (abs_max.float() / 127.0).clamp(min=1e-30)
|
| 151 |
+
q = _round_int8(x / _int8_scale_for_math(scale, x))
|
| 152 |
+
return q, scale
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
def quantize_int8_convrot_weight(weight: torch.Tensor, group_size: int) -> tuple[torch.Tensor, torch.Tensor]:
|
| 156 |
+
"""Offline ConvRot weight rotation followed by row-wise INT8 quantization.
|
| 157 |
+
|
| 158 |
+
The rotation is computed in float32 regardless of the weight dtype: rotating in
|
| 159 |
+
bf16 loses enough precision to shift ~9% of the int8 codes by ±1-2, while fp32
|
| 160 |
+
rotation reproduces ComfyUI's published pre-quantized checkpoints bit-exactly
|
| 161 |
+
(verified against the MiniMax-H3 INT8 ConvRot distribution).
|
| 162 |
+
|
| 163 |
+
Returns:
|
| 164 |
+
Tuple of (rotated quantized weight int8 [N, K], scale float32 [N, 1]).
|
| 165 |
+
"""
|
| 166 |
+
h = _build_hadamard(group_size, device=weight.device, dtype=torch.float32)
|
| 167 |
+
weight_rot = _rotate_weight(weight.to(torch.float32), h, group_size)
|
| 168 |
+
return quantize_int8_rowwise(weight_rot)
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
def dequantize_int8_convrot_weight(q: torch.Tensor, scale: torch.Tensor, group_size: int) -> torch.Tensor:
|
| 172 |
+
"""Dequantize INT8 ConvRot weights and rotate them back to the original basis."""
|
| 173 |
+
h = _build_hadamard(group_size, device=q.device, dtype=torch.float32)
|
| 174 |
+
return _rotate_weight(q.float() * scale, h, group_size)
|
| 175 |
+
|
| 176 |
+
|
| 177 |
+
# =============================================================================
|
| 178 |
+
# Triton kernels (from comfy_kitchen/backends/triton/quantization.py INT8 section:
|
| 179 |
+
# per-row activation quantization + INT8 GEMM with fused dequantization epilogue)
|
| 180 |
+
# =============================================================================
|
| 181 |
+
|
| 182 |
+
if HAS_TRITON:
|
| 183 |
+
|
| 184 |
+
@triton.jit
|
| 185 |
+
def _quantize_rowwise_kernel(
|
| 186 |
+
x_ptr, # Input pointer (FP16/BF16)
|
| 187 |
+
y_ptr, # Output pointer (INT8)
|
| 188 |
+
s_ptr, # Scale pointer (FP32)
|
| 189 |
+
n_elements, # Number of columns
|
| 190 |
+
block_size: tl.constexpr,
|
| 191 |
+
input_dtype_code: tl.constexpr,
|
| 192 |
+
):
|
| 193 |
+
# Row index we are processing. Promote to int64 before the stride multiply:
|
| 194 |
+
# H3 packs video+audio+text into one ~90k-row sequence, so row*cols can
|
| 195 |
+
# exceed int32 (Triton wraps silently, corrupting the tail of the tensor).
|
| 196 |
+
row_idx = tl.program_id(0).to(tl.int64)
|
| 197 |
+
|
| 198 |
+
# Pointers to the start of the row
|
| 199 |
+
x_row_ptr = x_ptr + row_idx * n_elements
|
| 200 |
+
y_row_ptr = y_ptr + row_idx * n_elements
|
| 201 |
+
|
| 202 |
+
# 1. Compute Max Abs Value for the row
|
| 203 |
+
offsets = tl.arange(0, block_size)
|
| 204 |
+
mask = offsets < n_elements
|
| 205 |
+
|
| 206 |
+
# Load data
|
| 207 |
+
x = tl.load(x_row_ptr + offsets, mask=mask, other=0.0)
|
| 208 |
+
|
| 209 |
+
# Absolute value
|
| 210 |
+
abs_x = tl.abs(x)
|
| 211 |
+
|
| 212 |
+
# Reduction to find max
|
| 213 |
+
max_val = tl.max(abs_x, axis=0)
|
| 214 |
+
|
| 215 |
+
# 2. Compute Scale
|
| 216 |
+
scale = tl.maximum(max_val / 127.0, 1e-30)
|
| 217 |
+
|
| 218 |
+
# 3. Quantize
|
| 219 |
+
if input_dtype_code == 1:
|
| 220 |
+
q_f = (x / scale.to(tl.float16)).to(tl.float16)
|
| 221 |
+
elif input_dtype_code == 2:
|
| 222 |
+
q_f = (x / scale.to(tl.bfloat16)).to(tl.bfloat16)
|
| 223 |
+
else:
|
| 224 |
+
q_f = x / scale
|
| 225 |
+
|
| 226 |
+
# Round and Clamp
|
| 227 |
+
q_i = tl.clamp(libdevice.rint(q_f.to(tl.float32)), -128.0, 127.0).to(tl.int32)
|
| 228 |
+
|
| 229 |
+
# 4. Store
|
| 230 |
+
tl.store(y_row_ptr + offsets, q_i.to(tl.int8), mask=mask)
|
| 231 |
+
tl.store(s_ptr + row_idx, scale.to(tl.float32))
|
| 232 |
+
|
| 233 |
+
def triton_quantize_rowwise(x: torch.Tensor):
|
| 234 |
+
"""
|
| 235 |
+
Input: [Batch, Dim] (float16/bfloat16/float32)
|
| 236 |
+
Output: [Batch, Dim] (int8), [Batch, 1] (float32)
|
| 237 |
+
"""
|
| 238 |
+
rows, cols = x.shape
|
| 239 |
+
y = torch.empty_like(x, dtype=torch.int8)
|
| 240 |
+
s = torch.empty((rows, 1), device=x.device, dtype=torch.float32)
|
| 241 |
+
|
| 242 |
+
input_dtype_code = 1 if x.dtype == torch.float16 else 2 if x.dtype == torch.bfloat16 else 0
|
| 243 |
+
|
| 244 |
+
# Heuristic for block size
|
| 245 |
+
block_size = triton.next_power_of_2(cols)
|
| 246 |
+
if block_size < 128:
|
| 247 |
+
block_size = 128
|
| 248 |
+
|
| 249 |
+
grid = (rows,)
|
| 250 |
+
_quantize_rowwise_kernel[grid](
|
| 251 |
+
x,
|
| 252 |
+
y,
|
| 253 |
+
s,
|
| 254 |
+
cols,
|
| 255 |
+
block_size=block_size,
|
| 256 |
+
input_dtype_code=input_dtype_code,
|
| 257 |
+
)
|
| 258 |
+
return y, s
|
| 259 |
+
|
| 260 |
+
@triton.autotune(
|
| 261 |
+
configs=[
|
| 262 |
+
triton.Config({"block_m": 128, "block_n": 256, "block_k": 64, "group_size_m": 8}, num_stages=3, num_warps=8),
|
| 263 |
+
triton.Config({"block_m": 64, "block_n": 256, "block_k": 32, "group_size_m": 8}, num_stages=4, num_warps=4),
|
| 264 |
+
triton.Config({"block_m": 128, "block_n": 128, "block_k": 32, "group_size_m": 8}, num_stages=4, num_warps=4),
|
| 265 |
+
triton.Config({"block_m": 128, "block_n": 64, "block_k": 32, "group_size_m": 8}, num_stages=4, num_warps=4),
|
| 266 |
+
triton.Config({"block_m": 64, "block_n": 128, "block_k": 32, "group_size_m": 8}, num_stages=4, num_warps=4),
|
| 267 |
+
triton.Config({"block_m": 128, "block_n": 32, "block_k": 32, "group_size_m": 8}, num_stages=4, num_warps=4),
|
| 268 |
+
],
|
| 269 |
+
key=["m", "n", "k"],
|
| 270 |
+
)
|
| 271 |
+
@triton.jit
|
| 272 |
+
def _int8_matmul_dequant_kernel(
|
| 273 |
+
# Pointers
|
| 274 |
+
a_ptr,
|
| 275 |
+
b_ptr,
|
| 276 |
+
c_ptr,
|
| 277 |
+
a_scale_ptr,
|
| 278 |
+
b_scale_ptr,
|
| 279 |
+
bias_ptr,
|
| 280 |
+
# Matrix Dimensions
|
| 281 |
+
m,
|
| 282 |
+
n,
|
| 283 |
+
k,
|
| 284 |
+
# Strides
|
| 285 |
+
stride_am,
|
| 286 |
+
stride_ak,
|
| 287 |
+
stride_bk,
|
| 288 |
+
stride_bn,
|
| 289 |
+
stride_cm,
|
| 290 |
+
stride_cn,
|
| 291 |
+
# Meta-parameters
|
| 292 |
+
block_m: tl.constexpr,
|
| 293 |
+
block_n: tl.constexpr,
|
| 294 |
+
block_k: tl.constexpr,
|
| 295 |
+
group_size_m: tl.constexpr,
|
| 296 |
+
has_bias: tl.constexpr,
|
| 297 |
+
):
|
| 298 |
+
"""
|
| 299 |
+
Computes: C = ((A * B) * (scale_a * scale_b)) + bias
|
| 300 |
+
A: [m, k] int8
|
| 301 |
+
B: [n, k] int8 (Transposed physically or logically via strides)
|
| 302 |
+
"""
|
| 303 |
+
pid = tl.program_id(axis=0)
|
| 304 |
+
num_pid_m = tl.cdiv(m, block_m)
|
| 305 |
+
num_pid_n = tl.cdiv(n, block_n)
|
| 306 |
+
num_pid_in_group = group_size_m * num_pid_n
|
| 307 |
+
group_id = pid // num_pid_in_group
|
| 308 |
+
first_pid_m = group_id * group_size_m
|
| 309 |
+
actual_group_size_m = min(num_pid_m - first_pid_m, group_size_m)
|
| 310 |
+
pid_m = first_pid_m + (pid % actual_group_size_m)
|
| 311 |
+
pid_n = (pid % num_pid_in_group) // actual_group_size_m
|
| 312 |
+
|
| 313 |
+
# 1. Prepare Pointers for A and B
|
| 314 |
+
offs_am = (pid_m * block_m + tl.arange(0, block_m)) % m
|
| 315 |
+
offs_bn = (pid_n * block_n + tl.arange(0, block_n)) % n
|
| 316 |
+
offs_k = tl.arange(0, block_k)
|
| 317 |
+
|
| 318 |
+
# row * stride must be int64: at H3 scale m*stride_am and m*stride_cm exceed
|
| 319 |
+
# int32 (e.g. fc1 output m~90k x n=28672), and Triton wraps i32 silently
|
| 320 |
+
offs_am_i64 = offs_am.to(tl.int64)
|
| 321 |
+
a_ptrs = a_ptr + (offs_am_i64[:, None] * stride_am + offs_k[None, :] * stride_ak)
|
| 322 |
+
b_ptrs = b_ptr + (offs_k[:, None] * stride_bk + offs_bn[None, :] * stride_bn)
|
| 323 |
+
|
| 324 |
+
# 2. Main Loop (Accumulate in Int32)
|
| 325 |
+
accumulator = tl.zeros((block_m, block_n), dtype=tl.int32)
|
| 326 |
+
|
| 327 |
+
for k_idx in range(0, tl.cdiv(k, block_k)):
|
| 328 |
+
# Load chunks
|
| 329 |
+
a = tl.load(a_ptrs, mask=offs_k[None, :] < k - k_idx * block_k, other=0)
|
| 330 |
+
b = tl.load(b_ptrs, mask=offs_k[:, None] < k - k_idx * block_k, other=0)
|
| 331 |
+
|
| 332 |
+
# Matrix Multiply
|
| 333 |
+
accumulator += tl.dot(a, b)
|
| 334 |
+
|
| 335 |
+
# Advance pointers
|
| 336 |
+
a_ptrs += block_k * stride_ak
|
| 337 |
+
b_ptrs += block_k * stride_bk
|
| 338 |
+
|
| 339 |
+
# 3. Fused Epilogue (Dequantize & Bias)
|
| 340 |
+
scale_a = tl.load(a_scale_ptr + offs_am) # Vector [BLOCK_M]
|
| 341 |
+
scale_b = tl.load(b_scale_ptr)
|
| 342 |
+
|
| 343 |
+
c = accumulator.to(tl.float32)
|
| 344 |
+
total_scale = scale_a[:, None] * scale_b
|
| 345 |
+
c = c * total_scale
|
| 346 |
+
|
| 347 |
+
if has_bias:
|
| 348 |
+
bias = tl.load(bias_ptr + offs_bn) # Vector [BLOCK_N]
|
| 349 |
+
c = c + bias[None, :]
|
| 350 |
+
|
| 351 |
+
# 4. Store Result
|
| 352 |
+
c_ptrs = c_ptr + stride_cm * offs_am_i64[:, None] + stride_cn * offs_bn[None, :]
|
| 353 |
+
c_mask = (offs_am[:, None] < m) & (offs_bn[None, :] < n)
|
| 354 |
+
tl.store(c_ptrs, c, mask=c_mask)
|
| 355 |
+
|
| 356 |
+
@triton.autotune(
|
| 357 |
+
configs=[
|
| 358 |
+
triton.Config({"block_m": 128, "block_n": 256, "block_k": 64, "group_size_m": 8}, num_stages=3, num_warps=8),
|
| 359 |
+
triton.Config({"block_m": 64, "block_n": 256, "block_k": 32, "group_size_m": 8}, num_stages=4, num_warps=4),
|
| 360 |
+
triton.Config({"block_m": 128, "block_n": 128, "block_k": 32, "group_size_m": 8}, num_stages=4, num_warps=4),
|
| 361 |
+
triton.Config({"block_m": 128, "block_n": 64, "block_k": 32, "group_size_m": 8}, num_stages=4, num_warps=4),
|
| 362 |
+
triton.Config({"block_m": 64, "block_n": 128, "block_k": 32, "group_size_m": 8}, num_stages=4, num_warps=4),
|
| 363 |
+
triton.Config({"block_m": 128, "block_n": 32, "block_k": 32, "group_size_m": 8}, num_stages=4, num_warps=4),
|
| 364 |
+
],
|
| 365 |
+
key=["m", "n", "k"],
|
| 366 |
+
)
|
| 367 |
+
@triton.jit
|
| 368 |
+
def _int8_matmul_dequant_per_row_kernel(
|
| 369 |
+
# Pointers
|
| 370 |
+
a_ptr,
|
| 371 |
+
b_ptr,
|
| 372 |
+
c_ptr,
|
| 373 |
+
a_scale_ptr,
|
| 374 |
+
b_scale_ptr,
|
| 375 |
+
bias_ptr,
|
| 376 |
+
# Matrix Dimensions
|
| 377 |
+
m,
|
| 378 |
+
n,
|
| 379 |
+
k,
|
| 380 |
+
# Strides
|
| 381 |
+
stride_am,
|
| 382 |
+
stride_ak,
|
| 383 |
+
stride_bk,
|
| 384 |
+
stride_bn,
|
| 385 |
+
stride_cm,
|
| 386 |
+
stride_cn,
|
| 387 |
+
# Meta-parameters
|
| 388 |
+
block_m: tl.constexpr,
|
| 389 |
+
block_n: tl.constexpr,
|
| 390 |
+
block_k: tl.constexpr,
|
| 391 |
+
group_size_m: tl.constexpr,
|
| 392 |
+
has_bias: tl.constexpr,
|
| 393 |
+
):
|
| 394 |
+
"""
|
| 395 |
+
Computes: C = ((A * B) * (scale_a[:, None] * scale_b[None, :])) + bias
|
| 396 |
+
A: [m, k] int8, scale_a: [m, 1] per-row activation scales
|
| 397 |
+
B: [n, k] int8, scale_b: [n, 1] per-row weight scales
|
| 398 |
+
"""
|
| 399 |
+
pid = tl.program_id(axis=0)
|
| 400 |
+
num_pid_m = tl.cdiv(m, block_m)
|
| 401 |
+
num_pid_n = tl.cdiv(n, block_n)
|
| 402 |
+
num_pid_in_group = group_size_m * num_pid_n
|
| 403 |
+
group_id = pid // num_pid_in_group
|
| 404 |
+
first_pid_m = group_id * group_size_m
|
| 405 |
+
actual_group_size_m = min(num_pid_m - first_pid_m, group_size_m)
|
| 406 |
+
pid_m = first_pid_m + (pid % actual_group_size_m)
|
| 407 |
+
pid_n = (pid % num_pid_in_group) // actual_group_size_m
|
| 408 |
+
|
| 409 |
+
# 1. Prepare Pointers for A and B
|
| 410 |
+
offs_am = (pid_m * block_m + tl.arange(0, block_m)) % m
|
| 411 |
+
offs_bn = (pid_n * block_n + tl.arange(0, block_n)) % n
|
| 412 |
+
offs_k = tl.arange(0, block_k)
|
| 413 |
+
|
| 414 |
+
# row * stride must be int64: at H3 scale m*stride_am and m*stride_cm exceed
|
| 415 |
+
# int32 (e.g. fc1 output m~90k x n=28672), and Triton wraps i32 silently
|
| 416 |
+
offs_am_i64 = offs_am.to(tl.int64)
|
| 417 |
+
a_ptrs = a_ptr + (offs_am_i64[:, None] * stride_am + offs_k[None, :] * stride_ak)
|
| 418 |
+
b_ptrs = b_ptr + (offs_k[:, None] * stride_bk + offs_bn[None, :] * stride_bn)
|
| 419 |
+
|
| 420 |
+
# 2. Main Loop (Accumulate in Int32)
|
| 421 |
+
accumulator = tl.zeros((block_m, block_n), dtype=tl.int32)
|
| 422 |
+
|
| 423 |
+
for k_idx in range(0, tl.cdiv(k, block_k)):
|
| 424 |
+
a = tl.load(a_ptrs, mask=offs_k[None, :] < k - k_idx * block_k, other=0)
|
| 425 |
+
b = tl.load(b_ptrs, mask=offs_k[:, None] < k - k_idx * block_k, other=0)
|
| 426 |
+
accumulator += tl.dot(a, b)
|
| 427 |
+
a_ptrs += block_k * stride_ak
|
| 428 |
+
b_ptrs += block_k * stride_bk
|
| 429 |
+
|
| 430 |
+
# 3. Fused Epilogue (Dequantize & Bias)
|
| 431 |
+
scale_a = tl.load(a_scale_ptr + offs_am) # Vector [BLOCK_M]
|
| 432 |
+
scale_b = tl.load(b_scale_ptr + offs_bn) # Vector [BLOCK_N]
|
| 433 |
+
|
| 434 |
+
c = accumulator.to(tl.float32)
|
| 435 |
+
total_scale = scale_a[:, None] * scale_b[None, :]
|
| 436 |
+
c = c * total_scale
|
| 437 |
+
|
| 438 |
+
if has_bias:
|
| 439 |
+
bias = tl.load(bias_ptr + offs_bn)
|
| 440 |
+
c = c + bias[None, :]
|
| 441 |
+
|
| 442 |
+
# 4. Store Result
|
| 443 |
+
c_ptrs = c_ptr + stride_cm * offs_am_i64[:, None] + stride_cn * offs_bn[None, :]
|
| 444 |
+
c_mask = (offs_am[:, None] < m) & (offs_bn[None, :] < n)
|
| 445 |
+
tl.store(c_ptrs, c, mask=c_mask)
|
| 446 |
+
|
| 447 |
+
def int8_linear(
|
| 448 |
+
x: torch.Tensor,
|
| 449 |
+
weight: torch.Tensor,
|
| 450 |
+
weight_scale: torch.Tensor,
|
| 451 |
+
bias: torch.Tensor | None = None,
|
| 452 |
+
out_dtype: torch.dtype = torch.bfloat16,
|
| 453 |
+
convrot: bool = False,
|
| 454 |
+
convrot_groupsize: int = 256,
|
| 455 |
+
) -> torch.Tensor:
|
| 456 |
+
"""INT8 linear layer using fused Triton kernel.
|
| 457 |
+
|
| 458 |
+
Quantizes x dynamically per-row, uses tensorwise or per-channel weight scale.
|
| 459 |
+
|
| 460 |
+
Args:
|
| 461 |
+
x: Input tensor [..., K].
|
| 462 |
+
weight: INT8 weight tensor [N, K].
|
| 463 |
+
weight_scale: Weight scale.
|
| 464 |
+
bias: Optional bias [N].
|
| 465 |
+
out_dtype: Output dtype.
|
| 466 |
+
convrot: If True, apply online activation rotation.
|
| 467 |
+
convrot_groupsize: Group size for Hadamard rotation.
|
| 468 |
+
|
| 469 |
+
Returns:
|
| 470 |
+
Result tensor [..., N].
|
| 471 |
+
"""
|
| 472 |
+
orig_shape = x.shape
|
| 473 |
+
x_2d = x.reshape(-1, x.shape[-1])
|
| 474 |
+
|
| 475 |
+
m, k = x_2d.shape
|
| 476 |
+
n = weight.shape[0]
|
| 477 |
+
|
| 478 |
+
# Quantize input per-row using fused or normal quantization kernel
|
| 479 |
+
if convrot:
|
| 480 |
+
h = _build_hadamard(convrot_groupsize, device=x.device, dtype=x.dtype)
|
| 481 |
+
x_rotated = _rotate_activation(x_2d, h, convrot_groupsize)
|
| 482 |
+
x_int8, x_scale = triton_quantize_rowwise(x_rotated)
|
| 483 |
+
else:
|
| 484 |
+
x_int8, x_scale = triton_quantize_rowwise(x_2d)
|
| 485 |
+
|
| 486 |
+
output = torch.empty((m, n), device=x.device, dtype=out_dtype)
|
| 487 |
+
|
| 488 |
+
is_per_channel = False
|
| 489 |
+
if not isinstance(weight_scale, torch.Tensor):
|
| 490 |
+
weight_scale = torch.tensor([weight_scale], device=x.device, dtype=torch.float32)
|
| 491 |
+
elif weight_scale.numel() == 1:
|
| 492 |
+
weight_scale = weight_scale.reshape(1)
|
| 493 |
+
else:
|
| 494 |
+
is_per_channel = True
|
| 495 |
+
weight_scale = weight_scale.reshape(n).contiguous()
|
| 496 |
+
|
| 497 |
+
has_bias = bias is not None
|
| 498 |
+
bias_ptr = bias if has_bias else x
|
| 499 |
+
|
| 500 |
+
def grid(meta):
|
| 501 |
+
return (triton.cdiv(m, meta["block_m"]) * triton.cdiv(n, meta["block_n"]),)
|
| 502 |
+
|
| 503 |
+
if is_per_channel:
|
| 504 |
+
_int8_matmul_dequant_per_row_kernel[grid](
|
| 505 |
+
a_ptr=x_int8,
|
| 506 |
+
b_ptr=weight,
|
| 507 |
+
c_ptr=output,
|
| 508 |
+
a_scale_ptr=x_scale,
|
| 509 |
+
b_scale_ptr=weight_scale,
|
| 510 |
+
bias_ptr=bias_ptr,
|
| 511 |
+
m=m,
|
| 512 |
+
n=n,
|
| 513 |
+
k=k,
|
| 514 |
+
stride_am=x_int8.stride(0),
|
| 515 |
+
stride_ak=x_int8.stride(1),
|
| 516 |
+
stride_bk=weight.stride(1),
|
| 517 |
+
stride_bn=weight.stride(0),
|
| 518 |
+
stride_cm=output.stride(0),
|
| 519 |
+
stride_cn=output.stride(1),
|
| 520 |
+
has_bias=has_bias,
|
| 521 |
+
)
|
| 522 |
+
else:
|
| 523 |
+
_int8_matmul_dequant_kernel[grid](
|
| 524 |
+
a_ptr=x_int8,
|
| 525 |
+
b_ptr=weight,
|
| 526 |
+
c_ptr=output,
|
| 527 |
+
a_scale_ptr=x_scale,
|
| 528 |
+
b_scale_ptr=weight_scale,
|
| 529 |
+
bias_ptr=bias_ptr,
|
| 530 |
+
m=m,
|
| 531 |
+
n=n,
|
| 532 |
+
k=k,
|
| 533 |
+
stride_am=x_int8.stride(0),
|
| 534 |
+
stride_ak=x_int8.stride(1),
|
| 535 |
+
stride_bk=weight.stride(1),
|
| 536 |
+
stride_bn=weight.stride(0),
|
| 537 |
+
stride_cm=output.stride(0),
|
| 538 |
+
stride_cn=output.stride(1),
|
| 539 |
+
has_bias=has_bias,
|
| 540 |
+
)
|
| 541 |
+
|
| 542 |
+
return output.reshape(*orig_shape[:-1], n)
|
| 543 |
+
|
| 544 |
+
else:
|
| 545 |
+
triton_quantize_rowwise = None
|
| 546 |
+
int8_linear = None
|
third_party/musubi_tuner/modules/convrot_int8_utils.py
ADDED
|
@@ -0,0 +1,451 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""ConvRot INT8 optimization for frozen base weights during LoRA training.
|
| 2 |
+
|
| 3 |
+
Mirrors the structure of fp8_optimization_utils.py: quantize target Linear weights at
|
| 4 |
+
load time, store the INT8 weight (in the rotated basis) under the original ``.weight``
|
| 5 |
+
key and the per-channel scale under a sibling ``.scale_weight`` key, then monkey-patch
|
| 6 |
+
the matching ``nn.Linear`` modules' ``forward``.
|
| 7 |
+
|
| 8 |
+
Unlike the fp8 path, the forward goes through a custom ``torch.autograd.Function``:
|
| 9 |
+
the fused Triton kernel (rotation + dynamic row-wise INT8 quantization + INT8 GEMM
|
| 10 |
+
with dequantization epilogue) has no autograd support, and the base weight is frozen
|
| 11 |
+
so only grad_x is needed in backward: grad_x = rotate(g @ W_rot).
|
| 12 |
+
|
| 13 |
+
Keeping the module an ``nn.Linear`` (patched forward, INT8 ``.weight``) is load-bearing:
|
| 14 |
+
LoRA targets modules by class name "Linear", block swap streams ``module.weight.data``
|
| 15 |
+
of Linear-named modules, and compile exclusion also keys on the class name.
|
| 16 |
+
"""
|
| 17 |
+
|
| 18 |
+
import math
|
| 19 |
+
import os
|
| 20 |
+
from collections.abc import Iterable
|
| 21 |
+
from pathlib import Path
|
| 22 |
+
from typing import Dict, List, Optional, Sequence, Union
|
| 23 |
+
|
| 24 |
+
import torch
|
| 25 |
+
import torch.nn as nn
|
| 26 |
+
import torch.nn.functional as F
|
| 27 |
+
|
| 28 |
+
import logging
|
| 29 |
+
|
| 30 |
+
from tqdm import tqdm
|
| 31 |
+
|
| 32 |
+
from musubi_tuner.modules.comfy_quant_utils import (
|
| 33 |
+
COMFY_QUANT_SUFFIX,
|
| 34 |
+
COMFY_WEIGHT_SCALE_SUFFIX,
|
| 35 |
+
decode_comfy_quant_spec,
|
| 36 |
+
)
|
| 37 |
+
from musubi_tuner.modules.convrot_int8_kernels import (
|
| 38 |
+
HAS_TRITON,
|
| 39 |
+
_build_hadamard,
|
| 40 |
+
_rotate_activation,
|
| 41 |
+
int8_linear,
|
| 42 |
+
quantize_int8_convrot_weight,
|
| 43 |
+
)
|
| 44 |
+
from musubi_tuner.utils.safetensors_utils import MemoryEfficientSafeOpen, TensorWeightAdapter, WeightTransformHooks
|
| 45 |
+
from musubi_tuner.utils.device_utils import clean_memory_on_device
|
| 46 |
+
|
| 47 |
+
logger = logging.getLogger(__name__)
|
| 48 |
+
logging.basicConfig(level=logging.INFO)
|
| 49 |
+
|
| 50 |
+
CONVROT_GROUPSIZE = 256
|
| 51 |
+
|
| 52 |
+
# ComfyUI pre-quantized ConvRot layers: `.weight` (int8, rotated basis) +
|
| 53 |
+
# `.weight_scale` (fp32 [N, 1]) + `.comfy_quant` (uint8 bytes of a JSON spec).
|
| 54 |
+
COMFY_QUANT_FORMAT_INT8 = "int8_tensorwise"
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def _is_power_of_4(value: int) -> bool:
|
| 58 |
+
return value >= 4 and (value & (value - 1)) == 0 and math.log(value, 4) % 1 == 0
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def select_convrot_groupsize(in_features: int, allowed_groupsizes: Sequence[int]) -> Optional[int]:
|
| 62 |
+
"""Pick the largest allowed group size that divides ``in_features``, or None.
|
| 63 |
+
|
| 64 |
+
The regular Hadamard construction only exists for power-of-4 sizes, so allowing a
|
| 65 |
+
smaller fallback (e.g. 64) extends coverage to layers whose in_features is not a
|
| 66 |
+
multiple of 256 (H3's adaln_proj: 2688 = 42 * 64). This rule reproduces ComfyUI's
|
| 67 |
+
published group size choices exactly.
|
| 68 |
+
"""
|
| 69 |
+
for groupsize in sorted(allowed_groupsizes, reverse=True):
|
| 70 |
+
if in_features % groupsize == 0:
|
| 71 |
+
return groupsize
|
| 72 |
+
return None
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def quantize_weight_convrot(key: str, tensor: torch.Tensor, allowed_groupsizes: Sequence[int] = (CONVROT_GROUPSIZE,)):
|
| 76 |
+
"""Quantize a single weight tensor with ConvRot INT8, or return None if not applicable.
|
| 77 |
+
|
| 78 |
+
The quantization is always done with the eager implementation: it is deterministic
|
| 79 |
+
and independent of triton availability, so the resulting state dict is identical
|
| 80 |
+
across environments.
|
| 81 |
+
|
| 82 |
+
Returns:
|
| 83 |
+
(quantized int8 [N, K] in rotated basis, scale float32 [N, 1], groupsize), or
|
| 84 |
+
None if the tensor is not a 2D weight or its in_features is not divisible by
|
| 85 |
+
any allowed groupsize.
|
| 86 |
+
"""
|
| 87 |
+
if tensor.ndim != 2:
|
| 88 |
+
logger.info(f"Skipping ConvRot INT8 for {key}: not a 2D weight (ndim={tensor.ndim})")
|
| 89 |
+
return None
|
| 90 |
+
groupsize = select_convrot_groupsize(tensor.shape[1], allowed_groupsizes)
|
| 91 |
+
if groupsize is None:
|
| 92 |
+
logger.info(f"Skipping ConvRot INT8 for {key}: in_features {tensor.shape[1]} not divisible by any of {allowed_groupsizes}")
|
| 93 |
+
return None
|
| 94 |
+
quantized_weight, scale_tensor = quantize_int8_convrot_weight(tensor, groupsize)
|
| 95 |
+
return quantized_weight, scale_tensor, groupsize
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
def parse_comfy_quant_spec(key: str, tensor: torch.Tensor) -> dict:
|
| 99 |
+
"""Decode and validate a ``.comfy_quant`` JSON spec tensor (uint8 bytes).
|
| 100 |
+
|
| 101 |
+
Only ConvRot INT8 (``int8_tensorwise`` + ``convrot``) is supported; other formats
|
| 102 |
+
(e.g. nvfp4) raise with a clear message.
|
| 103 |
+
"""
|
| 104 |
+
spec = decode_comfy_quant_spec(key, tensor)
|
| 105 |
+
quant_format = spec.get("format")
|
| 106 |
+
if quant_format != COMFY_QUANT_FORMAT_INT8 or not spec.get("convrot"):
|
| 107 |
+
raise ValueError(
|
| 108 |
+
f"Unsupported comfy_quant format for {key}: {spec}. Only ConvRot INT8"
|
| 109 |
+
f' ("{COMFY_QUANT_FORMAT_INT8}" with "convrot": true) is supported.'
|
| 110 |
+
f" / {key} の comfy_quant 形式はサポートされていません。ConvRot INT8 のみ対応しています。"
|
| 111 |
+
)
|
| 112 |
+
groupsize = spec.get("convrot_groupsize")
|
| 113 |
+
if not isinstance(groupsize, int) or not _is_power_of_4(groupsize):
|
| 114 |
+
raise ValueError(f"Invalid convrot_groupsize for {key}: {groupsize!r} (must be a power of 4, e.g. 64 or 256)")
|
| 115 |
+
return spec
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
def canonicalize_convrot_int8_key(key: str) -> str:
|
| 119 |
+
"""Map the ComfyUI ``.weight_scale`` suffix to the Musubi ``.scale_weight`` layout."""
|
| 120 |
+
if key.endswith(COMFY_WEIGHT_SCALE_SUFFIX):
|
| 121 |
+
return key[: -len(COMFY_WEIGHT_SCALE_SUFFIX)] + ".scale_weight"
|
| 122 |
+
return key
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
def has_comfy_quant_tensors(files: Iterable[Union[str, Path]], *, disable_numpy_memmap: bool = False) -> bool:
|
| 126 |
+
"""Header-only probe: does the checkpoint declare ComfyUI pre-quantized layers?
|
| 127 |
+
|
| 128 |
+
Routing only — validation and conversion of the ``.comfy_quant`` triples happen in
|
| 129 |
+
``ConvRotInt8Quantizer.load_and_quantize``.
|
| 130 |
+
"""
|
| 131 |
+
for file in files:
|
| 132 |
+
with MemoryEfficientSafeOpen(str(file), disable_numpy_memmap=disable_numpy_memmap) as f:
|
| 133 |
+
if any(key.endswith(COMFY_QUANT_SUFFIX) for key in f.keys()):
|
| 134 |
+
return True
|
| 135 |
+
return False
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
class ConvRotInt8Quantizer:
|
| 139 |
+
"""Strategy object that streams safetensors files and quantizes target weights.
|
| 140 |
+
|
| 141 |
+
Passed as ``quantizer`` to ``load_safetensors_with_lora_and_fp8``; carries its own
|
| 142 |
+
streaming loader so the fp8 path stays untouched.
|
| 143 |
+
|
| 144 |
+
Two sources are supported per tensor, decided on the fly:
|
| 145 |
+
|
| 146 |
+
- bf16/fp16/fp32 target weights are dynamically quantized (rotation + row-wise INT8),
|
| 147 |
+
picking the largest ``allowed_groupsizes`` entry that divides in_features.
|
| 148 |
+
- Pre-quantized ComfyUI layers (int8 ``.weight`` + ``.weight_scale`` + ``.comfy_quant``)
|
| 149 |
+
are converted in place: the scale is renamed to ``.scale_weight`` and the group size
|
| 150 |
+
is taken from the JSON spec. The file dictates which layers these are, independent
|
| 151 |
+
of the target/exclude patterns.
|
| 152 |
+
|
| 153 |
+
``module_groupsizes`` maps module paths to their group size after loading; pass it to
|
| 154 |
+
``apply_convrot_int8_monkey_patch`` so mixed group sizes dispatch correctly.
|
| 155 |
+
"""
|
| 156 |
+
|
| 157 |
+
def __init__(
|
| 158 |
+
self,
|
| 159 |
+
target_layer_keys: Optional[List[str]] = None,
|
| 160 |
+
exclude_layer_keys: Optional[List[str]] = None,
|
| 161 |
+
allowed_groupsizes: Sequence[int] = (CONVROT_GROUPSIZE,),
|
| 162 |
+
):
|
| 163 |
+
for groupsize in allowed_groupsizes:
|
| 164 |
+
if not _is_power_of_4(groupsize):
|
| 165 |
+
raise ValueError(f"ConvRot group sizes must be powers of 4, got {groupsize}")
|
| 166 |
+
self.target_layer_keys = target_layer_keys
|
| 167 |
+
self.exclude_layer_keys = exclude_layer_keys
|
| 168 |
+
self.allowed_groupsizes = tuple(allowed_groupsizes)
|
| 169 |
+
self.module_groupsizes: Dict[str, int] = {}
|
| 170 |
+
|
| 171 |
+
def is_target_key(self, key: str) -> bool:
|
| 172 |
+
is_target = (self.target_layer_keys is None or any(pattern in key for pattern in self.target_layer_keys)) and key.endswith(
|
| 173 |
+
".weight"
|
| 174 |
+
)
|
| 175 |
+
is_excluded = self.exclude_layer_keys is not None and any(pattern in key for pattern in self.exclude_layer_keys)
|
| 176 |
+
return is_target and not is_excluded
|
| 177 |
+
|
| 178 |
+
def load_and_quantize(
|
| 179 |
+
self,
|
| 180 |
+
model_files: List[str],
|
| 181 |
+
calc_device: Union[str, torch.device, None],
|
| 182 |
+
move_to_device: bool = False,
|
| 183 |
+
weight_hook: Optional[callable] = None,
|
| 184 |
+
disable_numpy_memmap: bool = False,
|
| 185 |
+
weight_transform_hooks: Optional[WeightTransformHooks] = None,
|
| 186 |
+
) -> dict:
|
| 187 |
+
"""Load state dict from safetensors files, quantizing target weights to ConvRot INT8.
|
| 188 |
+
|
| 189 |
+
Same streaming contract as load_safetensors_with_fp8_optimization: the LoRA merge
|
| 190 |
+
weight_hook runs on the raw (bf16) weight before quantization. Pre-quantized
|
| 191 |
+
ComfyUI ConvRot layers are converted to the Musubi layout instead; they cannot be
|
| 192 |
+
combined with a LoRA merge hook (INT8 weights cannot be merged into).
|
| 193 |
+
"""
|
| 194 |
+
optimized_count = 0
|
| 195 |
+
prequantized_count = 0
|
| 196 |
+
state_dict = {}
|
| 197 |
+
prequantized_groupsizes: Dict[str, int] = {} # spans all shards
|
| 198 |
+
for model_file in model_files:
|
| 199 |
+
with MemoryEfficientSafeOpen(model_file, disable_numpy_memmap=disable_numpy_memmap) as original_f:
|
| 200 |
+
f = TensorWeightAdapter(weight_transform_hooks, original_f) if weight_transform_hooks is not None else original_f
|
| 201 |
+
|
| 202 |
+
keys = f.keys()
|
| 203 |
+
|
| 204 |
+
# Pre-scan the tiny `.comfy_quant` spec tensors so the per-module group size
|
| 205 |
+
# is known before the (possibly earlier-iterated) weight/scale keys arrive.
|
| 206 |
+
for key in keys:
|
| 207 |
+
if key.endswith(COMFY_QUANT_SUFFIX):
|
| 208 |
+
module_path = key[: -len(COMFY_QUANT_SUFFIX)]
|
| 209 |
+
spec = parse_comfy_quant_spec(key, f.get_tensor(key))
|
| 210 |
+
prequantized_groupsizes[module_path] = spec["convrot_groupsize"]
|
| 211 |
+
if prequantized_groupsizes and weight_hook is not None:
|
| 212 |
+
raise ValueError(
|
| 213 |
+
f"Cannot merge LoRA weights into pre-quantized ConvRot INT8 checkpoint {model_file}."
|
| 214 |
+
" Use the original BF16 weights to merge LoRA at load time, or apply the LoRA at runtime."
|
| 215 |
+
f" / 事前量子化済みConvRot INT8チェックポイント {model_file} にはLoRAをマージできません。"
|
| 216 |
+
"BF16の元重みを使用してロード時マージするか、LoRAを実行時適用してください。"
|
| 217 |
+
)
|
| 218 |
+
|
| 219 |
+
for key in tqdm(keys, desc=f"Loading {os.path.basename(model_file)}", unit="key"):
|
| 220 |
+
if key.endswith(COMFY_QUANT_SUFFIX):
|
| 221 |
+
continue # consumed in the pre-scan, not a model tensor
|
| 222 |
+
|
| 223 |
+
value = f.get_tensor(key)
|
| 224 |
+
original_device = value.device # usually cpu
|
| 225 |
+
passthrough_device = calc_device if (calc_device is not None and move_to_device) else original_device
|
| 226 |
+
|
| 227 |
+
if key.endswith(COMFY_WEIGHT_SCALE_SUFFIX):
|
| 228 |
+
module_path = key[: -len(COMFY_WEIGHT_SCALE_SUFFIX)]
|
| 229 |
+
if module_path not in prequantized_groupsizes:
|
| 230 |
+
raise ValueError(f"Found {key} without a matching {module_path}{COMFY_QUANT_SUFFIX} spec")
|
| 231 |
+
if value.dtype is not torch.float32:
|
| 232 |
+
raise ValueError(f"Pre-quantized ConvRot scale {key} must be F32, got {value.dtype}")
|
| 233 |
+
# rename to the Musubi layout; the fp32 [N, 1] shape is shared as-is
|
| 234 |
+
state_dict[module_path + ".scale_weight"] = value.to(passthrough_device)
|
| 235 |
+
continue
|
| 236 |
+
|
| 237 |
+
module_path = key[: -len(".weight")] if key.endswith(".weight") else None
|
| 238 |
+
if module_path is not None and module_path in prequantized_groupsizes:
|
| 239 |
+
if value.dtype != torch.int8:
|
| 240 |
+
raise ValueError(f"Pre-quantized ConvRot layer {key} must be int8, got {value.dtype}")
|
| 241 |
+
groupsize = prequantized_groupsizes[module_path]
|
| 242 |
+
if value.shape[1] % groupsize != 0:
|
| 243 |
+
raise ValueError(
|
| 244 |
+
f"Pre-quantized ConvRot layer {key}: in_features {value.shape[1]} not divisible by"
|
| 245 |
+
f" group size {groupsize}"
|
| 246 |
+
)
|
| 247 |
+
self.module_groupsizes[module_path] = groupsize
|
| 248 |
+
state_dict[key] = value.to(passthrough_device)
|
| 249 |
+
prequantized_count += 1
|
| 250 |
+
continue
|
| 251 |
+
|
| 252 |
+
if module_path is not None and value.dtype.itemsize == 1:
|
| 253 |
+
raise ValueError(
|
| 254 |
+
f"Layer {key} is already in {value.dtype} format but has no {COMFY_QUANT_SUFFIX} spec."
|
| 255 |
+
" Only ComfyUI ConvRot INT8 pre-quantized checkpoints or fp16/bf16/float32 weights are"
|
| 256 |
+
f" supported. / レイヤー {key} は既に{value.dtype}形式ですが {COMFY_QUANT_SUFFIX} がありません。"
|
| 257 |
+
"ComfyUI ConvRot INT8形式の事前量子化済み重み、またはFP16/BF16/Float32の重みを使用してください。"
|
| 258 |
+
)
|
| 259 |
+
|
| 260 |
+
if weight_hook is not None:
|
| 261 |
+
value = weight_hook(key, value, keep_on_calc_device=(calc_device is not None))
|
| 262 |
+
|
| 263 |
+
if not self.is_target_key(key):
|
| 264 |
+
state_dict[key] = value.to(passthrough_device)
|
| 265 |
+
continue
|
| 266 |
+
|
| 267 |
+
if calc_device is not None:
|
| 268 |
+
value = value.to(calc_device)
|
| 269 |
+
|
| 270 |
+
result = quantize_weight_convrot(key, value, self.allowed_groupsizes)
|
| 271 |
+
if result is None:
|
| 272 |
+
# leave the layer unquantized (bf16)
|
| 273 |
+
if not move_to_device:
|
| 274 |
+
value = value.to(original_device)
|
| 275 |
+
state_dict[key] = value
|
| 276 |
+
continue
|
| 277 |
+
quantized_weight, scale_tensor, groupsize = result
|
| 278 |
+
self.module_groupsizes[module_path] = groupsize
|
| 279 |
+
|
| 280 |
+
scale_key = key.replace(".weight", ".scale_weight")
|
| 281 |
+
assert key != scale_key, "weight key and scale key must be different"
|
| 282 |
+
|
| 283 |
+
if not move_to_device:
|
| 284 |
+
quantized_weight = quantized_weight.to(original_device)
|
| 285 |
+
|
| 286 |
+
# scale stays float32 [N, 1]: the Triton epilogue and the backward want fp32,
|
| 287 |
+
# and the shape maps 1:1 to ComfyUI's `weight_scale`
|
| 288 |
+
state_dict[key] = quantized_weight
|
| 289 |
+
state_dict[scale_key] = scale_tensor.to(device=quantized_weight.device)
|
| 290 |
+
|
| 291 |
+
optimized_count += 1
|
| 292 |
+
|
| 293 |
+
if calc_device is not None and optimized_count % 10 == 0:
|
| 294 |
+
clean_memory_on_device(calc_device)
|
| 295 |
+
|
| 296 |
+
# every declared `.comfy_quant` spec must have received its int8 weight and fp32
|
| 297 |
+
# [N, 1] scale; a partial triple would otherwise assign-load into an unpatched Linear
|
| 298 |
+
for module_path in prequantized_groupsizes:
|
| 299 |
+
weight_key = module_path + ".weight"
|
| 300 |
+
scale_key = module_path + ".scale_weight"
|
| 301 |
+
missing = [key for key in (weight_key, scale_key) if key not in state_dict]
|
| 302 |
+
if missing:
|
| 303 |
+
raise ValueError(f"Pre-quantized ConvRot layer {module_path} is missing tensors {missing}")
|
| 304 |
+
expected_scale_shape = (state_dict[weight_key].shape[0], 1)
|
| 305 |
+
if tuple(state_dict[scale_key].shape) != expected_scale_shape:
|
| 306 |
+
raise ValueError(
|
| 307 |
+
f"Pre-quantized ConvRot layer {module_path}: scale shape must be {expected_scale_shape},"
|
| 308 |
+
f" got {tuple(state_dict[scale_key].shape)}"
|
| 309 |
+
)
|
| 310 |
+
|
| 311 |
+
logger.info(
|
| 312 |
+
f"Number of ConvRot INT8 Linear layers: {optimized_count} dynamically quantized,"
|
| 313 |
+
f" {prequantized_count} loaded pre-quantized"
|
| 314 |
+
)
|
| 315 |
+
return state_dict
|
| 316 |
+
|
| 317 |
+
|
| 318 |
+
class ConvRotInt8LinearFn(torch.autograd.Function):
|
| 319 |
+
@staticmethod
|
| 320 |
+
@torch.amp.custom_fwd(device_type="cuda")
|
| 321 |
+
def forward(ctx, x, wq, w_scale, bias, groupsize, bwd_mode):
|
| 322 |
+
# x: [..., K] bf16/fp16, wq: [N, K] int8 (rotated basis), w_scale: [N, 1] fp32
|
| 323 |
+
# F.linear casts its inputs to the autocast dtype under autocast; the fused kernel
|
| 324 |
+
# bypasses F.linear, so replicate that here. In K2 the fp32 modulation adds promote
|
| 325 |
+
# the activations to fp32, and downstream flash-attn only accepts fp16/bf16.
|
| 326 |
+
if torch.is_autocast_enabled(x.device.type):
|
| 327 |
+
cast_dtype = torch.get_autocast_dtype(x.device.type)
|
| 328 |
+
x = x.to(cast_dtype)
|
| 329 |
+
if bias is not None:
|
| 330 |
+
bias = bias.to(cast_dtype)
|
| 331 |
+
if HAS_TRITON and x.is_cuda:
|
| 332 |
+
out = int8_linear(x, wq, w_scale.reshape(-1), bias, x.dtype, True, groupsize)
|
| 333 |
+
else:
|
| 334 |
+
# eager fallback: rotation + transient dequantized matmul, no activation quantization
|
| 335 |
+
h = _build_hadamard(groupsize, device=x.device, dtype=x.dtype)
|
| 336 |
+
x_rot = _rotate_activation(x, h, groupsize)
|
| 337 |
+
w_rot = wq.to(x.dtype) * w_scale.reshape(-1, 1).to(x.dtype)
|
| 338 |
+
out = F.linear(x_rot, w_rot, bias)
|
| 339 |
+
# wq/w_scale are live buffers, so saving them adds no activation memory; x is not
|
| 340 |
+
# saved (base is frozen, no grad_weight needed)
|
| 341 |
+
ctx.save_for_backward(wq, w_scale)
|
| 342 |
+
ctx.groupsize = groupsize
|
| 343 |
+
ctx.bwd_mode = bwd_mode
|
| 344 |
+
ctx.bias_needs_grad = bias is not None and bias.requires_grad
|
| 345 |
+
return out
|
| 346 |
+
|
| 347 |
+
@staticmethod
|
| 348 |
+
@torch.amp.custom_bwd(device_type="cuda")
|
| 349 |
+
def backward(ctx, grad_out):
|
| 350 |
+
wq, w_scale = ctx.saved_tensors
|
| 351 |
+
gs = ctx.groupsize
|
| 352 |
+
g2d = grad_out.reshape(-1, grad_out.shape[-1]) # [M, N]
|
| 353 |
+
|
| 354 |
+
grad_x = None
|
| 355 |
+
if ctx.needs_input_grad[0]:
|
| 356 |
+
# grad_x = g @ W = g @ (W_rot R) = rotate(g @ W_rot), R = block-diag Hadamard
|
| 357 |
+
if ctx.bwd_mode == "int8":
|
| 358 |
+
if not g2d.is_cuda:
|
| 359 |
+
raise RuntimeError("ConvRot INT8 backward mode 'int8' requires CUDA tensors")
|
| 360 |
+
# fold per-channel weight scale into g, then reuse the fused Triton GEMM
|
| 361 |
+
# (row-wise quant of g + int8 GEMM + dequant epilogue in one pipeline).
|
| 362 |
+
# transient int8 transpose of wq: [K, N], ~1 byte/param, freed after mm
|
| 363 |
+
g_scaled = g2d * w_scale.reshape(1, -1).to(g2d.dtype)
|
| 364 |
+
one = torch.ones(1, device=g2d.device, dtype=torch.float32)
|
| 365 |
+
gx_rot = int8_linear(g_scaled, wq.t().contiguous(), one, None, grad_out.dtype, False, gs)
|
| 366 |
+
else:
|
| 367 |
+
# transient bf16 dequant of the rotated weight (stays in rotated basis)
|
| 368 |
+
w_rot = wq.to(grad_out.dtype) * w_scale.reshape(-1, 1).to(grad_out.dtype)
|
| 369 |
+
gx_rot = g2d @ w_rot # [M, K]
|
| 370 |
+
h = _build_hadamard(gs, device=gx_rot.device, dtype=gx_rot.dtype)
|
| 371 |
+
grad_x = _rotate_activation(gx_rot, h, gs).reshape(*grad_out.shape[:-1], wq.shape[1])
|
| 372 |
+
|
| 373 |
+
grad_bias = g2d.sum(dim=0) if ctx.bias_needs_grad else None
|
| 374 |
+
return grad_x, None, None, grad_bias, None, None
|
| 375 |
+
|
| 376 |
+
|
| 377 |
+
def convrot_int8_linear_forward_patch(self: nn.Linear, x):
|
| 378 |
+
return ConvRotInt8LinearFn.apply(x, self.weight, self.scale_weight, self.bias, self._convrot_groupsize, self._convrot_bwd_mode)
|
| 379 |
+
|
| 380 |
+
|
| 381 |
+
def _validate_convrot_bwd_mode(bwd_mode: str) -> None:
|
| 382 |
+
if bwd_mode not in ("bf16", "int8"):
|
| 383 |
+
raise ValueError(f"Unsupported ConvRot INT8 backward mode: {bwd_mode}")
|
| 384 |
+
if bwd_mode == "int8" and not HAS_TRITON:
|
| 385 |
+
raise ValueError("ConvRot INT8 backward mode 'int8' requires triton. Install triton (triton-windows on Windows).")
|
| 386 |
+
if not HAS_TRITON:
|
| 387 |
+
logger.warning(
|
| 388 |
+
"triton is not available: ConvRot INT8 falls back to transient dequantization in forward."
|
| 389 |
+
" Weight VRAM is still reduced, but there is no speedup. Install triton (triton-windows on Windows)"
|
| 390 |
+
" for the fused INT8 kernels."
|
| 391 |
+
)
|
| 392 |
+
|
| 393 |
+
|
| 394 |
+
def _patch_convrot_int8_linear(module: nn.Linear, groupsize: int, bwd_mode: str) -> None:
|
| 395 |
+
module._convrot_groupsize = groupsize
|
| 396 |
+
module._convrot_bwd_mode = bwd_mode
|
| 397 |
+
module.forward = convrot_int8_linear_forward_patch.__get__(module, type(module))
|
| 398 |
+
|
| 399 |
+
|
| 400 |
+
def apply_convrot_int8_monkey_patch(
|
| 401 |
+
model,
|
| 402 |
+
optimized_state_dict,
|
| 403 |
+
bwd_mode: str = "bf16",
|
| 404 |
+
groupsize: int = CONVROT_GROUPSIZE,
|
| 405 |
+
groupsize_map: Optional[Dict[str, int]] = None,
|
| 406 |
+
):
|
| 407 |
+
"""
|
| 408 |
+
Apply monkey patching to a model using a ConvRot INT8 optimized state dict.
|
| 409 |
+
|
| 410 |
+
Args:
|
| 411 |
+
model (nn.Module): Model instance to patch
|
| 412 |
+
optimized_state_dict (dict): state dict produced by ConvRotInt8Quantizer
|
| 413 |
+
bwd_mode (str): "bf16" (transient dequant, safe default) or "int8" (quantizes
|
| 414 |
+
gradients, faster, requires triton)
|
| 415 |
+
groupsize (int): ConvRot group size for modules not listed in ``groupsize_map``
|
| 416 |
+
groupsize_map (Optional[Dict[str, int]]): per-module group sizes
|
| 417 |
+
(``ConvRotInt8Quantizer.module_groupsizes``); needed when a checkpoint mixes
|
| 418 |
+
group sizes (e.g. H3: 256 for attn/mlp, 64 for adaln_proj)
|
| 419 |
+
|
| 420 |
+
Returns:
|
| 421 |
+
nn.Module: The patched model (same instance, modified in-place)
|
| 422 |
+
"""
|
| 423 |
+
_validate_convrot_bwd_mode(bwd_mode)
|
| 424 |
+
|
| 425 |
+
scale_keys = [k for k in optimized_state_dict.keys() if k.endswith(".scale_weight")]
|
| 426 |
+
|
| 427 |
+
patched_module_paths = set()
|
| 428 |
+
scale_shape_info = {}
|
| 429 |
+
for scale_key in scale_keys:
|
| 430 |
+
module_path = scale_key.rsplit(".scale_weight", 1)[0]
|
| 431 |
+
patched_module_paths.add(module_path)
|
| 432 |
+
scale_shape_info[module_path] = optimized_state_dict[scale_key].shape
|
| 433 |
+
|
| 434 |
+
patched_paths = set()
|
| 435 |
+
for name, module in model.named_modules():
|
| 436 |
+
if isinstance(module, nn.Linear) and name in patched_module_paths:
|
| 437 |
+
# register the scale_weight as a buffer to load the state_dict
|
| 438 |
+
module.register_buffer("scale_weight", torch.ones(scale_shape_info[name], dtype=torch.float32))
|
| 439 |
+
module_groupsize = groupsize_map.get(name, groupsize) if groupsize_map is not None else groupsize
|
| 440 |
+
_patch_convrot_int8_linear(module, module_groupsize, bwd_mode)
|
| 441 |
+
|
| 442 |
+
patched_paths.add(name)
|
| 443 |
+
|
| 444 |
+
unmatched = sorted(patched_module_paths - patched_paths)
|
| 445 |
+
if unmatched:
|
| 446 |
+
raise ValueError(f"ConvRot INT8 state dict declares missing module {unmatched[0]} (total {len(unmatched)} unmatched)")
|
| 447 |
+
|
| 448 |
+
model.is_convrot_int8 = True
|
| 449 |
+
model.convrot_int8_layer_count = len(patched_paths)
|
| 450 |
+
logger.info(f"Number of ConvRot INT8 monkey-patched Linear layers: {len(patched_paths)}")
|
| 451 |
+
return model
|
third_party/musubi_tuner/modules/custom_offloading_utils.py
ADDED
|
@@ -0,0 +1,1184 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from collections import defaultdict
|
| 2 |
+
from concurrent.futures import ThreadPoolExecutor
|
| 3 |
+
from contextlib import contextmanager
|
| 4 |
+
from dataclasses import dataclass
|
| 5 |
+
import gc
|
| 6 |
+
import time
|
| 7 |
+
from typing import Optional
|
| 8 |
+
import torch
|
| 9 |
+
import torch.nn as nn
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
# Keep these functions here for portability, and private to avoid confusion with the ones in device_utils.py
|
| 13 |
+
def _clean_memory_on_device(device: torch.device):
|
| 14 |
+
r"""
|
| 15 |
+
Clean memory on the specified device, will be called from training scripts.
|
| 16 |
+
"""
|
| 17 |
+
gc.collect()
|
| 18 |
+
|
| 19 |
+
# device may "cuda" or "cuda:0", so we need to check the type of device
|
| 20 |
+
if device.type == "cuda":
|
| 21 |
+
torch.cuda.empty_cache()
|
| 22 |
+
if device.type == "xpu":
|
| 23 |
+
torch.xpu.empty_cache()
|
| 24 |
+
if device.type == "mps":
|
| 25 |
+
torch.mps.empty_cache()
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def _synchronize_device(device: torch.device):
|
| 29 |
+
if device.type == "cuda":
|
| 30 |
+
torch.cuda.synchronize()
|
| 31 |
+
elif device.type == "xpu":
|
| 32 |
+
torch.xpu.synchronize()
|
| 33 |
+
elif device.type == "mps":
|
| 34 |
+
torch.mps.synchronize()
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def swap_weight_devices_no_cuda(device: torch.device, layer_to_cpu: nn.Module, layer_to_cuda: nn.Module):
|
| 38 |
+
"""
|
| 39 |
+
not tested
|
| 40 |
+
"""
|
| 41 |
+
assert layer_to_cpu.__class__ == layer_to_cuda.__class__
|
| 42 |
+
|
| 43 |
+
weight_swap_jobs = []
|
| 44 |
+
for module_to_cpu, module_to_cuda in zip(layer_to_cpu.modules(), layer_to_cuda.modules()):
|
| 45 |
+
if hasattr(module_to_cpu, "weight") and module_to_cpu.weight is not None:
|
| 46 |
+
weight_swap_jobs.append((module_to_cpu, module_to_cuda, module_to_cpu.weight.data, module_to_cuda.weight.data))
|
| 47 |
+
|
| 48 |
+
# device to cpu
|
| 49 |
+
for module_to_cpu, module_to_cuda, cuda_data_view, cpu_data_view in weight_swap_jobs:
|
| 50 |
+
module_to_cpu.weight.data = cuda_data_view.data.to("cpu", non_blocking=True)
|
| 51 |
+
|
| 52 |
+
_synchronize_device(device)
|
| 53 |
+
|
| 54 |
+
# cpu to device
|
| 55 |
+
for module_to_cpu, module_to_cuda, cuda_data_view, cpu_data_view in weight_swap_jobs:
|
| 56 |
+
cuda_data_view.copy_(module_to_cuda.weight.data, non_blocking=True)
|
| 57 |
+
module_to_cuda.weight.data = cuda_data_view
|
| 58 |
+
|
| 59 |
+
_synchronize_device(device)
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def weighs_to_device(layer: nn.Module, device: torch.device):
|
| 63 |
+
for module in layer.modules():
|
| 64 |
+
if hasattr(module, "weight") and module.weight is not None and module.__class__.__name__.endswith("Linear"):
|
| 65 |
+
module.weight.data = module.weight.data.to(device, non_blocking=device.type != "cpu")
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
def default_swap_tensor_selector(block: nn.Module) -> list[tuple[nn.Module, str]]:
|
| 69 |
+
"""Default streaming selection rule: the ``weight`` of every ``*Linear`` module.
|
| 70 |
+
|
| 71 |
+
Same rule as ``ModelOffloader.swap_weight_devices_cuda``. An offloader can be given a
|
| 72 |
+
different selector to also stream tensors that a quantization scheme hangs off the same
|
| 73 |
+
modules (per-group scales etc.); each entry is ``(module, attribute_name)`` where the
|
| 74 |
+
attribute is a parameter or a registered buffer of the module.
|
| 75 |
+
"""
|
| 76 |
+
jobs = []
|
| 77 |
+
for _, module in block.named_modules():
|
| 78 |
+
if hasattr(module, "weight") and module.weight is not None and module.__class__.__name__.endswith("Linear"):
|
| 79 |
+
jobs.append((module, "weight"))
|
| 80 |
+
return jobs
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
def attach_forward_streaming_hooks(offloader, blocks: list[nn.Module]) -> list:
|
| 84 |
+
"""Drive a block-swap offloader from forward hooks.
|
| 85 |
+
|
| 86 |
+
For models whose forward loop does not call the offloader itself (e.g. a Hugging Face
|
| 87 |
+
text encoder): each block waits for its own weights before running and releases/prefetches
|
| 88 |
+
after it finishes. Forward-only offloaders handle the pass boundary (wrap-around preload)
|
| 89 |
+
internally. Hooks must be attached before any per-call hooks so the prefetch of the last
|
| 90 |
+
block runs first. Returns the hook handles so callers can detach them.
|
| 91 |
+
"""
|
| 92 |
+
handles = []
|
| 93 |
+
for index, block in enumerate(blocks):
|
| 94 |
+
|
| 95 |
+
def _wait_hook(module, args, _index=index):
|
| 96 |
+
del module, args
|
| 97 |
+
offloader.wait_for_block(_index)
|
| 98 |
+
|
| 99 |
+
def _submit_hook(module, args, output, _index=index):
|
| 100 |
+
del module, args, output
|
| 101 |
+
offloader.submit_move_blocks_forward(blocks, _index)
|
| 102 |
+
|
| 103 |
+
handles.append(block.register_forward_pre_hook(_wait_hook))
|
| 104 |
+
handles.append(block.register_forward_hook(_submit_hook))
|
| 105 |
+
return handles
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
@dataclass
|
| 109 |
+
class BlockSwapConfig:
|
| 110 |
+
"""
|
| 111 |
+
Construction policy for a block-swap offloader, assembled once by the training/inference script and
|
| 112 |
+
passed through each architecture's ``enable_block_swap`` to ``create_offloader``.
|
| 113 |
+
|
| 114 |
+
It holds everything the offloader constructor needs that is *not* architecture-specific. The
|
| 115 |
+
architecture-specific arguments (the block-type label, the block list, and its counts) are supplied by
|
| 116 |
+
``enable_block_swap`` per block list. Adding a new offloader knob (or a whole new offloader type) means
|
| 117 |
+
adding a field here and a branch in ``create_offloader`` -- the per-architecture ``enable_block_swap``
|
| 118 |
+
signatures stay ``(blocks_to_swap, config)`` and never change.
|
| 119 |
+
"""
|
| 120 |
+
|
| 121 |
+
device: torch.device
|
| 122 |
+
supports_backward: bool
|
| 123 |
+
use_pinned_memory: bool = False
|
| 124 |
+
h2d_only: bool = False # frozen-base (LoRA / LoHa / LoKr) only: H2D-only streaming, no device->host copy
|
| 125 |
+
ring_size: int = 2 # (h2d_only) number of GPU ring buffers for streamed blocks; 2 = double buffering
|
| 126 |
+
debug: bool = False
|
| 127 |
+
|
| 128 |
+
@classmethod
|
| 129 |
+
def from_args(cls, args, device: torch.device, supports_backward: bool) -> "BlockSwapConfig":
|
| 130 |
+
"""Build from a parsed-args namespace, tolerating scripts whose parser lacks the optional knobs."""
|
| 131 |
+
h2d_only = getattr(args, "block_swap_h2d_only", False)
|
| 132 |
+
|
| 133 |
+
# H2D-only streams frozen weights into a reused GPU ring buffer and loads them with an in-place copy_,
|
| 134 |
+
# which bumps the autograd version of the saved weight view. Gradient checkpointing re-reads the weight
|
| 135 |
+
# at recompute time (a short, safe window), so the version stays consistent; without it the backward
|
| 136 |
+
# pass fails with an opaque "modified by an inplace operation ... version N expected M" error that does
|
| 137 |
+
# not point at the offloader. Fail early with an actionable message instead. (Inference / forward-only
|
| 138 |
+
# has no backward, so it is unaffected.)
|
| 139 |
+
if h2d_only and supports_backward and not getattr(args, "gradient_checkpointing", False):
|
| 140 |
+
raise ValueError(
|
| 141 |
+
"--block_swap_h2d_only requires --gradient_checkpointing for training. H2D-only block swap streams"
|
| 142 |
+
" frozen weights through a reused GPU ring buffer, which advances the autograd version of weights"
|
| 143 |
+
" saved for backward; gradient checkpointing re-reads them at recompute time and avoids this."
|
| 144 |
+
" / --block_swap_h2d_only は学習時に --gradient_checkpointing が必須です(リングバッファの上書きで"
|
| 145 |
+
"backward 用に保存された重みの version が進むため。gradient checkpointing は再計算時に読み直すので回避できます)。"
|
| 146 |
+
)
|
| 147 |
+
|
| 148 |
+
ring_size = getattr(args, "block_swap_ring_size", 2)
|
| 149 |
+
if ring_size < 1:
|
| 150 |
+
raise ValueError("--block_swap_ring_size must be >= 1")
|
| 151 |
+
|
| 152 |
+
return cls(
|
| 153 |
+
device=device,
|
| 154 |
+
supports_backward=supports_backward,
|
| 155 |
+
use_pinned_memory=getattr(args, "use_pinned_memory_for_block_swap", False),
|
| 156 |
+
h2d_only=h2d_only,
|
| 157 |
+
ring_size=ring_size,
|
| 158 |
+
)
|
| 159 |
+
|
| 160 |
+
|
| 161 |
+
def create_offloader(block_type: str, blocks: list[nn.Module], num_blocks: int, blocks_to_swap: int, config: BlockSwapConfig):
|
| 162 |
+
"""
|
| 163 |
+
Create the block-swap offloader for one block list. This is the single place that selects the offloader
|
| 164 |
+
implementation from ``config``; ``enable_block_swap`` only supplies the per-block-list arguments, so a new
|
| 165 |
+
offloader type plugs in here without touching any architecture.
|
| 166 |
+
"""
|
| 167 |
+
if config.h2d_only:
|
| 168 |
+
# H2D-only streaming for frozen-base (LoRA) training: keep a CPU master, copy Host->Device only.
|
| 169 |
+
return LoRAStreamOffloader(
|
| 170 |
+
block_type,
|
| 171 |
+
blocks,
|
| 172 |
+
num_blocks,
|
| 173 |
+
blocks_to_swap,
|
| 174 |
+
config.supports_backward,
|
| 175 |
+
config.device,
|
| 176 |
+
ring_size=config.ring_size,
|
| 177 |
+
use_pinned_memory=config.use_pinned_memory,
|
| 178 |
+
debug=config.debug,
|
| 179 |
+
)
|
| 180 |
+
return ModelOffloader(
|
| 181 |
+
block_type,
|
| 182 |
+
blocks,
|
| 183 |
+
num_blocks,
|
| 184 |
+
blocks_to_swap,
|
| 185 |
+
config.supports_backward,
|
| 186 |
+
config.device,
|
| 187 |
+
config.use_pinned_memory,
|
| 188 |
+
debug=config.debug,
|
| 189 |
+
)
|
| 190 |
+
|
| 191 |
+
|
| 192 |
+
class Offloader:
|
| 193 |
+
"""
|
| 194 |
+
common offloading class
|
| 195 |
+
"""
|
| 196 |
+
|
| 197 |
+
def __init__(
|
| 198 |
+
self,
|
| 199 |
+
block_type: str,
|
| 200 |
+
num_blocks: int,
|
| 201 |
+
blocks_to_swap: int,
|
| 202 |
+
device: torch.device,
|
| 203 |
+
use_pinned_memory: bool = False,
|
| 204 |
+
debug: bool = False,
|
| 205 |
+
):
|
| 206 |
+
self.block_type = block_type
|
| 207 |
+
self.num_blocks = num_blocks
|
| 208 |
+
self.blocks_to_swap = blocks_to_swap
|
| 209 |
+
self.device = device
|
| 210 |
+
self.use_pinned_memory = use_pinned_memory
|
| 211 |
+
|
| 212 |
+
# check if debug is enabled from os environment variable
|
| 213 |
+
if not debug:
|
| 214 |
+
import os
|
| 215 |
+
|
| 216 |
+
debug = os.getenv("MUSUBI_TUNER_OFFLOADER_DEBUG", "0") == "1"
|
| 217 |
+
|
| 218 |
+
self.debug = debug
|
| 219 |
+
self.debug_block_count = 0
|
| 220 |
+
|
| 221 |
+
self.thread_pool = ThreadPoolExecutor(max_workers=1)
|
| 222 |
+
self.futures = {}
|
| 223 |
+
self.cuda_available = device.type == "cuda"
|
| 224 |
+
self.stream = torch.cuda.Stream(device=device) if self.cuda_available else None
|
| 225 |
+
|
| 226 |
+
# Staging buffers for cuda offloading without large pinned memory. These are pinned memory buffers to speed up the transfer between CPU and GPU
|
| 227 |
+
# We create one staging buffer per transfer direction (A: GPU to CPU, B: CPU to GPU)
|
| 228 |
+
self.staging_buffer_a = None
|
| 229 |
+
self.staging_buffer_b = None
|
| 230 |
+
|
| 231 |
+
# Pinned buffer for cuda offloading with pinned memory. We need only one pinned buffer per layer transfer
|
| 232 |
+
self.pinned_buffer = None
|
| 233 |
+
|
| 234 |
+
def swap_weight_devices_cuda(self, device: torch.device, layer_to_cpu: nn.Module, layer_to_cuda: nn.Module):
|
| 235 |
+
assert layer_to_cpu.__class__ == layer_to_cuda.__class__
|
| 236 |
+
|
| 237 |
+
debug_print = False
|
| 238 |
+
if self.debug:
|
| 239 |
+
debug_print = self.debug_block_count % 10 == 0
|
| 240 |
+
self.debug_block_count += 1
|
| 241 |
+
|
| 242 |
+
class Timer:
|
| 243 |
+
def __init__(self, enabled=False):
|
| 244 |
+
self.enabled = enabled
|
| 245 |
+
self.totals = defaultdict(float)
|
| 246 |
+
self.start_time = time.perf_counter()
|
| 247 |
+
|
| 248 |
+
@contextmanager
|
| 249 |
+
def section(self, name):
|
| 250 |
+
if not self.enabled:
|
| 251 |
+
yield
|
| 252 |
+
return
|
| 253 |
+
t0 = time.perf_counter()
|
| 254 |
+
try:
|
| 255 |
+
yield
|
| 256 |
+
finally:
|
| 257 |
+
self.totals[name] += time.perf_counter() - t0
|
| 258 |
+
|
| 259 |
+
T = Timer(enabled=debug_print)
|
| 260 |
+
|
| 261 |
+
weight_swap_jobs = []
|
| 262 |
+
|
| 263 |
+
# This is not working for all cases (e.g. SD3), so we need to find the corresponding modules. kept here for reference:
|
| 264 |
+
# for module_to_cpu, module_to_cuda in zip(layer_to_cpu.modules(), layer_to_cuda.modules()):
|
| 265 |
+
# print(module_to_cpu.__class__, module_to_cuda.__class__)
|
| 266 |
+
# if hasattr(module_to_cpu, "weight") and module_to_cpu.weight is not None:
|
| 267 |
+
# weight_swap_jobs.append((module_to_cpu, module_to_cuda, module_to_cpu.weight.data, module_to_cuda.weight.data))
|
| 268 |
+
|
| 269 |
+
with T.section("find modules"):
|
| 270 |
+
modules_to_cpu = {k: v for k, v in layer_to_cpu.named_modules()}
|
| 271 |
+
for module_to_cuda_name, module_to_cuda in layer_to_cuda.named_modules():
|
| 272 |
+
if (
|
| 273 |
+
hasattr(module_to_cuda, "weight")
|
| 274 |
+
and module_to_cuda.weight is not None
|
| 275 |
+
and module_to_cuda.__class__.__name__.endswith("Linear")
|
| 276 |
+
):
|
| 277 |
+
module_to_cpu = modules_to_cpu.get(module_to_cuda_name, None)
|
| 278 |
+
if module_to_cpu is not None and module_to_cpu.weight.shape == module_to_cuda.weight.shape:
|
| 279 |
+
weight_swap_jobs.append(
|
| 280 |
+
(module_to_cpu, module_to_cuda, module_to_cpu.weight.data, module_to_cuda.weight.data)
|
| 281 |
+
)
|
| 282 |
+
else:
|
| 283 |
+
if module_to_cuda.weight.data.device.type != device.type:
|
| 284 |
+
module_to_cuda.weight.data = module_to_cuda.weight.data.to(device)
|
| 285 |
+
|
| 286 |
+
with T.section("synchronize before swap"):
|
| 287 |
+
torch.cuda.current_stream().synchronize() # this prevents the illegal loss value by ensuring offloading layer's calculation is done
|
| 288 |
+
|
| 289 |
+
if not self.use_pinned_memory:
|
| 290 |
+
# Minimize using pinned memory for lower shared GPU RAM usage
|
| 291 |
+
stream = self.stream
|
| 292 |
+
with torch.cuda.stream(stream):
|
| 293 |
+
if self.staging_buffer_a is None:
|
| 294 |
+
# Create staging buffer as pinned memory (as shared GPU ram). We specify device for correct pinning on multi-GPU systems
|
| 295 |
+
self.staging_buffer_a = [
|
| 296 |
+
torch.empty_like(cuda_data_view, device="cpu").pin_memory(device=device)
|
| 297 |
+
for _, _, cuda_data_view, _ in weight_swap_jobs
|
| 298 |
+
]
|
| 299 |
+
self.staging_buffer_b = [
|
| 300 |
+
torch.empty_like(cuda_data_view, device="cpu").pin_memory(device=device)
|
| 301 |
+
for _, _, cuda_data_view, _ in weight_swap_jobs
|
| 302 |
+
]
|
| 303 |
+
|
| 304 |
+
# Copy weights to staging buffers and record events
|
| 305 |
+
event_b = None
|
| 306 |
+
for sbuf_a, sbuf_b, (module_to_cpu, module_to_cuda, cuda_data_view, cpu_data_view) in zip(
|
| 307 |
+
self.staging_buffer_a, self.staging_buffer_b, weight_swap_jobs
|
| 308 |
+
):
|
| 309 |
+
# CUDA to staging buffer A, non-blocking copy
|
| 310 |
+
event_a = torch.cuda.Event()
|
| 311 |
+
with T.section("cuda to staging A"):
|
| 312 |
+
sbuf_a.copy_(cuda_data_view.data, non_blocking=True)
|
| 313 |
+
event_a.record(stream)
|
| 314 |
+
|
| 315 |
+
# Wait for staging buffer B to be ready
|
| 316 |
+
if event_b is not None:
|
| 317 |
+
with T.section("wait staging B"):
|
| 318 |
+
event_b.synchronize() # synchronize is needed to wait CPU process. wait_event does not work here because it waits on GPU side only
|
| 319 |
+
|
| 320 |
+
# CPU to staging buffer B, CPU to pinned CPU, synchronous copy. Can overlap with CUDA to staging buffer A
|
| 321 |
+
with T.section("cpu to staging B"):
|
| 322 |
+
# Making this multithreaded does not help, and 'non_blocking=True' does not help either.
|
| 323 |
+
sbuf_b.copy_(module_to_cuda.weight.data) # BOTTLENECK
|
| 324 |
+
|
| 325 |
+
# Wait for staging buffer A to be ready, and CUDA data view can be reused
|
| 326 |
+
with T.section("wait staging A"):
|
| 327 |
+
event_a.synchronize()
|
| 328 |
+
|
| 329 |
+
# Staging buffer B to CUDA, non-blocking copy.
|
| 330 |
+
event_b = torch.cuda.Event()
|
| 331 |
+
with T.section("staging B to CUDA"):
|
| 332 |
+
cuda_data_view.copy_(sbuf_b, non_blocking=True)
|
| 333 |
+
event_b.record(stream)
|
| 334 |
+
|
| 335 |
+
# Staging buffer A to CPU, synchronous copy. Can overlap with staging buffer B to CUDA
|
| 336 |
+
with T.section("staging A to CPU"):
|
| 337 |
+
cpu_data_view.copy_(sbuf_a) # BOTTLENECK
|
| 338 |
+
|
| 339 |
+
for sbuf_a, sbuf_b, (module_to_cpu, module_to_cuda, cuda_data_view, cpu_data_view) in zip(
|
| 340 |
+
self.staging_buffer_a, self.staging_buffer_b, weight_swap_jobs
|
| 341 |
+
):
|
| 342 |
+
# Update references
|
| 343 |
+
module_to_cuda.weight.data = cuda_data_view
|
| 344 |
+
module_to_cpu.weight.data = cpu_data_view
|
| 345 |
+
|
| 346 |
+
sync_event = event_b # final sync event for CPU to CUDA copy
|
| 347 |
+
|
| 348 |
+
else:
|
| 349 |
+
# Use pinned memory for faster transfer between CPU and GPU, but it requires more memory
|
| 350 |
+
if self.pinned_buffer is None:
|
| 351 |
+
with torch.cuda.stream(self.stream):
|
| 352 |
+
# Create pinned buffer as pinned memory (as shared GPU ram). We specify device for correct pinning on multi-GPU systems
|
| 353 |
+
self.pinned_buffer = [
|
| 354 |
+
torch.empty_like(cuda_data_view, device="cpu").pin_memory(device=device)
|
| 355 |
+
for _, _, cuda_data_view, _ in weight_swap_jobs
|
| 356 |
+
]
|
| 357 |
+
self.stream.synchronize()
|
| 358 |
+
released_pinned_buffer = []
|
| 359 |
+
|
| 360 |
+
events = [torch.cuda.Event() for _ in weight_swap_jobs] # Waiting events for GPU to CPU non-blocking copy
|
| 361 |
+
|
| 362 |
+
# Copy weights to CPU
|
| 363 |
+
for event, module_pin_buf, (module_to_cpu, module_to_cuda, cuda_data_view, cpu_data_view) in zip(
|
| 364 |
+
events, self.pinned_buffer, weight_swap_jobs
|
| 365 |
+
):
|
| 366 |
+
# CUDA to CPU, non-blocking copy
|
| 367 |
+
with torch.cuda.stream(self.stream):
|
| 368 |
+
with T.section("cuda to cpu"):
|
| 369 |
+
module_pin_buf.copy_(cuda_data_view, non_blocking=True)
|
| 370 |
+
event.record(self.stream)
|
| 371 |
+
|
| 372 |
+
# CPU to CUDA
|
| 373 |
+
for event, (module_to_cpu, module_to_cuda, cuda_data_view, cpu_data_view) in zip(events, weight_swap_jobs):
|
| 374 |
+
with torch.cuda.stream(self.stream):
|
| 375 |
+
# Wait for cuda_data_view to be ready
|
| 376 |
+
with T.section("wait cpu"):
|
| 377 |
+
self.stream.wait_event(event)
|
| 378 |
+
|
| 379 |
+
# CPU to CUDA, non-blocking copy
|
| 380 |
+
with T.section("cpu to cuda"):
|
| 381 |
+
cuda_data_view.copy_(cpu_data_view, non_blocking=True)
|
| 382 |
+
|
| 383 |
+
# Update references
|
| 384 |
+
for module_pin_buf, (module_to_cpu, module_to_cuda, cuda_data_view, cpu_data_view) in zip(
|
| 385 |
+
self.pinned_buffer, weight_swap_jobs
|
| 386 |
+
):
|
| 387 |
+
module_to_cuda.weight.data = cuda_data_view
|
| 388 |
+
module_to_cpu.weight.data = module_pin_buf
|
| 389 |
+
released_pinned_buffer.append(cpu_data_view) # CPU data view can be reused as pinned buffer
|
| 390 |
+
|
| 391 |
+
# Reuse released pinned buffers
|
| 392 |
+
if not released_pinned_buffer[0].is_pinned():
|
| 393 |
+
# In first time, we need to create pinned buffers because offloaded weights are not pinned yet
|
| 394 |
+
with torch.cuda.stream(self.stream):
|
| 395 |
+
released_pinned_buffer = [
|
| 396 |
+
torch.empty_like(cuda_data_view, device="cpu").pin_memory(device=device)
|
| 397 |
+
for _, _, cuda_data_view, _ in weight_swap_jobs
|
| 398 |
+
]
|
| 399 |
+
self.pinned_buffer = released_pinned_buffer
|
| 400 |
+
|
| 401 |
+
sync_event = self.stream.record_event()
|
| 402 |
+
|
| 403 |
+
if debug_print:
|
| 404 |
+
print(f"[{self.block_type}] Weight swap timing at {self.debug_block_count - 1}:")
|
| 405 |
+
for name, total in T.totals.items():
|
| 406 |
+
print(f" {name}: {total * 1000:.2f}ms")
|
| 407 |
+
print(
|
| 408 |
+
f"Overall time: {(time.perf_counter() - T.start_time) * 1000:.2f}ms, total time in sections: {sum(T.totals.values()) * 1000:.2f}ms"
|
| 409 |
+
)
|
| 410 |
+
# print(
|
| 411 |
+
# f"[{self.block_type}] Swapped weights in {time.perf_counter() - start_time:.2f}s. Count of modules swapped: {len(weight_swap_jobs)}"
|
| 412 |
+
# )
|
| 413 |
+
|
| 414 |
+
return sync_event
|
| 415 |
+
|
| 416 |
+
def swap_weight_devices(self, block_to_cpu: nn.Module, block_to_cuda: nn.Module):
|
| 417 |
+
if self.cuda_available:
|
| 418 |
+
sync_event = self.swap_weight_devices_cuda(self.device, block_to_cpu, block_to_cuda)
|
| 419 |
+
else:
|
| 420 |
+
swap_weight_devices_no_cuda(self.device, block_to_cpu, block_to_cuda)
|
| 421 |
+
sync_event = None
|
| 422 |
+
return sync_event
|
| 423 |
+
|
| 424 |
+
def _submit_move_blocks(self, blocks, block_idx_to_cpu, block_idx_to_cuda):
|
| 425 |
+
def move_blocks(bidx_to_cpu, block_to_cpu, bidx_to_cuda, block_to_cuda):
|
| 426 |
+
if self.debug:
|
| 427 |
+
start_time = time.perf_counter()
|
| 428 |
+
print(
|
| 429 |
+
f"[{self.block_type}] Move block {bidx_to_cpu} to CPU and block {bidx_to_cuda} to {'CUDA' if self.cuda_available else 'device'}"
|
| 430 |
+
)
|
| 431 |
+
|
| 432 |
+
dev = self.device.index if self.device.index is not None else torch.cuda.current_device()
|
| 433 |
+
torch.cuda.set_device(dev)
|
| 434 |
+
|
| 435 |
+
sync_event = self.swap_weight_devices(block_to_cpu, block_to_cuda)
|
| 436 |
+
|
| 437 |
+
if self.debug:
|
| 438 |
+
print(
|
| 439 |
+
f"[{self.block_type}] Moved blocks {bidx_to_cpu} to CPU and {bidx_to_cuda} to {'CUDA' if self.cuda_available else 'device'} in {time.perf_counter() - start_time:.2f}s"
|
| 440 |
+
)
|
| 441 |
+
return bidx_to_cpu, bidx_to_cuda, sync_event
|
| 442 |
+
|
| 443 |
+
block_to_cpu = blocks[block_idx_to_cpu]
|
| 444 |
+
block_to_cuda = blocks[block_idx_to_cuda]
|
| 445 |
+
|
| 446 |
+
self.futures[block_idx_to_cuda] = self.thread_pool.submit(
|
| 447 |
+
move_blocks, block_idx_to_cpu, block_to_cpu, block_idx_to_cuda, block_to_cuda
|
| 448 |
+
)
|
| 449 |
+
|
| 450 |
+
def _wait_blocks_move(self, block_idx):
|
| 451 |
+
if block_idx not in self.futures:
|
| 452 |
+
return
|
| 453 |
+
|
| 454 |
+
if self.debug:
|
| 455 |
+
print(f"[{self.block_type}] Wait for block {block_idx}")
|
| 456 |
+
start_time = time.perf_counter()
|
| 457 |
+
|
| 458 |
+
future = self.futures.pop(block_idx)
|
| 459 |
+
_, bidx_to_cuda, sync_event = future.result()
|
| 460 |
+
|
| 461 |
+
assert block_idx == bidx_to_cuda, f"Block index mismatch: {block_idx} != {bidx_to_cuda}"
|
| 462 |
+
|
| 463 |
+
if self.cuda_available and sync_event is not None:
|
| 464 |
+
# this does not wait CPU side, so the log below should be immediate when pinned memory is used
|
| 465 |
+
torch.cuda.current_stream().wait_event(sync_event)
|
| 466 |
+
|
| 467 |
+
if self.debug:
|
| 468 |
+
print(f"[{self.block_type}] Waited for block {block_idx}: {time.perf_counter() - start_time:.2f}s")
|
| 469 |
+
|
| 470 |
+
|
| 471 |
+
class ModelOffloader(Offloader):
|
| 472 |
+
"""
|
| 473 |
+
supports forward offloading
|
| 474 |
+
"""
|
| 475 |
+
|
| 476 |
+
def __init__(
|
| 477 |
+
self,
|
| 478 |
+
block_type: str,
|
| 479 |
+
blocks: list[nn.Module],
|
| 480 |
+
num_blocks: int,
|
| 481 |
+
blocks_to_swap: int,
|
| 482 |
+
supports_backward: bool,
|
| 483 |
+
device: torch.device,
|
| 484 |
+
use_pinned_memory: bool = False,
|
| 485 |
+
debug: bool = False,
|
| 486 |
+
):
|
| 487 |
+
super().__init__(block_type, num_blocks, blocks_to_swap, device, use_pinned_memory, debug)
|
| 488 |
+
|
| 489 |
+
self.supports_backward = supports_backward
|
| 490 |
+
self.forward_only = not supports_backward # forward only offloading: can be changed to True for inference
|
| 491 |
+
|
| 492 |
+
if self.supports_backward:
|
| 493 |
+
# register backward hooks
|
| 494 |
+
self.remove_handles = []
|
| 495 |
+
for i, block in enumerate(blocks):
|
| 496 |
+
hook = self.create_backward_hook(blocks, i)
|
| 497 |
+
if hook is not None:
|
| 498 |
+
handle = block.register_full_backward_hook(hook)
|
| 499 |
+
self.remove_handles.append(handle)
|
| 500 |
+
|
| 501 |
+
def set_forward_only(self, forward_only: bool):
|
| 502 |
+
# switching must wait for all pending transfers
|
| 503 |
+
for block_idx in list(self.futures.keys()):
|
| 504 |
+
self._wait_blocks_move(block_idx)
|
| 505 |
+
|
| 506 |
+
self.forward_only = forward_only
|
| 507 |
+
|
| 508 |
+
def __del__(self):
|
| 509 |
+
if self.supports_backward:
|
| 510 |
+
for handle in self.remove_handles:
|
| 511 |
+
handle.remove()
|
| 512 |
+
|
| 513 |
+
def create_backward_hook(self, blocks: list[nn.Module], block_index: int) -> Optional[callable]:
|
| 514 |
+
# -1 for 0-based index
|
| 515 |
+
num_blocks_propagated = self.num_blocks - block_index - 1
|
| 516 |
+
swapping = num_blocks_propagated > 0 and num_blocks_propagated <= self.blocks_to_swap
|
| 517 |
+
waiting = block_index > 0 and block_index <= self.blocks_to_swap
|
| 518 |
+
|
| 519 |
+
if not swapping and not waiting:
|
| 520 |
+
return None
|
| 521 |
+
|
| 522 |
+
# create hook
|
| 523 |
+
block_idx_to_cpu = self.num_blocks - num_blocks_propagated
|
| 524 |
+
block_idx_to_cuda = self.blocks_to_swap - num_blocks_propagated
|
| 525 |
+
block_idx_to_wait = block_index - 1
|
| 526 |
+
|
| 527 |
+
def backward_hook(module, grad_input, grad_output):
|
| 528 |
+
if self.debug:
|
| 529 |
+
print(f"Backward hook for block {block_index}")
|
| 530 |
+
|
| 531 |
+
if swapping:
|
| 532 |
+
self._submit_move_blocks(blocks, block_idx_to_cpu, block_idx_to_cuda)
|
| 533 |
+
if waiting:
|
| 534 |
+
self._wait_blocks_move(block_idx_to_wait)
|
| 535 |
+
return None
|
| 536 |
+
|
| 537 |
+
return backward_hook
|
| 538 |
+
|
| 539 |
+
def prepare_block_devices_before_forward(self, blocks: list[nn.Module]):
|
| 540 |
+
if self.blocks_to_swap is None or self.blocks_to_swap == 0:
|
| 541 |
+
return
|
| 542 |
+
|
| 543 |
+
if self.debug:
|
| 544 |
+
print(f"[{self.block_type}] Prepare block devices before forward")
|
| 545 |
+
|
| 546 |
+
for b in blocks[0 : self.num_blocks - self.blocks_to_swap]:
|
| 547 |
+
b.to(self.device)
|
| 548 |
+
weighs_to_device(b, self.device) # make sure weights are on device
|
| 549 |
+
|
| 550 |
+
cpu_device = torch.device("cpu")
|
| 551 |
+
for b in blocks[self.num_blocks - self.blocks_to_swap :]:
|
| 552 |
+
b.to(self.device) # move block to device first. this makes sure that buffers (non weights) are on the device
|
| 553 |
+
weighs_to_device(b, cpu_device) # make sure weights are on cpu
|
| 554 |
+
|
| 555 |
+
_synchronize_device(self.device)
|
| 556 |
+
_clean_memory_on_device(self.device)
|
| 557 |
+
|
| 558 |
+
def wait_for_block(self, block_idx: int):
|
| 559 |
+
if self.blocks_to_swap is None or self.blocks_to_swap == 0:
|
| 560 |
+
return
|
| 561 |
+
self._wait_blocks_move(block_idx)
|
| 562 |
+
|
| 563 |
+
def submit_move_blocks_forward(self, blocks: list[nn.Module], block_idx: int):
|
| 564 |
+
# check if blocks_to_swap is enabled
|
| 565 |
+
if self.blocks_to_swap is None or self.blocks_to_swap == 0:
|
| 566 |
+
return
|
| 567 |
+
|
| 568 |
+
if not self.forward_only:
|
| 569 |
+
# if backward is enabled, we do not swap blocks in forward pass more than blocks_to_swap, because it should be on GPU
|
| 570 |
+
if block_idx >= self.blocks_to_swap:
|
| 571 |
+
return
|
| 572 |
+
block_idx_to_cpu = block_idx
|
| 573 |
+
block_idx_to_cuda = self.num_blocks - self.blocks_to_swap + block_idx
|
| 574 |
+
block_idx_to_cuda = block_idx_to_cuda % self.num_blocks # this does nothing for backward offloading
|
| 575 |
+
self._submit_move_blocks(blocks, block_idx_to_cpu, block_idx_to_cuda)
|
| 576 |
+
return
|
| 577 |
+
|
| 578 |
+
# We use two strategies here for forward-only offloading:
|
| 579 |
+
# 1. If blocks_to_swap is less than half of num_blocks, we swap the num_blocks blocks without wrapping around.
|
| 580 |
+
# This reduces the number of swaps, so it is especially useful for small blocks_to_swap or lightweight models like Qwen-Image
|
| 581 |
+
# 2. If blocks_to_swap is more than half of num_blocks, we swap the blocks with wrapping around.
|
| 582 |
+
# This is the common strategy used in most offloading implementations. It transfers all blocks in a wrapping manner.
|
| 583 |
+
# This is useful for large blocks_to_swap or heavyweight models like Wan/HunyuanVideo, where the transfer time is less significant compared to computation time.
|
| 584 |
+
|
| 585 |
+
# current block to swap out (to CPU)
|
| 586 |
+
block_idx_to_cpu = block_idx
|
| 587 |
+
|
| 588 |
+
if self.blocks_to_swap < (self.num_blocks // 2):
|
| 589 |
+
# strategy 1: no wrap around
|
| 590 |
+
# If the current block is in the middle blocks that are not swapped, do nothing
|
| 591 |
+
if self.blocks_to_swap <= block_idx < self.num_blocks - self.blocks_to_swap:
|
| 592 |
+
return
|
| 593 |
+
if block_idx < self.blocks_to_swap:
|
| 594 |
+
# move the next block to cuda
|
| 595 |
+
block_idx_to_cuda = (self.num_blocks - self.blocks_to_swap + block_idx) % self.num_blocks
|
| 596 |
+
else:
|
| 597 |
+
# move the previous block to cuda
|
| 598 |
+
block_idx_to_cuda = block_idx - (self.num_blocks - self.blocks_to_swap)
|
| 599 |
+
else:
|
| 600 |
+
# strategy 2: with wrap around
|
| 601 |
+
block_idx_to_cuda = self.num_blocks - self.blocks_to_swap + block_idx
|
| 602 |
+
block_idx_to_cuda = block_idx_to_cuda % self.num_blocks # this works for forward-only offloading
|
| 603 |
+
|
| 604 |
+
self._submit_move_blocks(blocks, block_idx_to_cpu, block_idx_to_cuda)
|
| 605 |
+
|
| 606 |
+
|
| 607 |
+
class _DirectCopier:
|
| 608 |
+
"""
|
| 609 |
+
Transfer engine for ``LoRAStreamOffloader`` when the masters are *pinned*: copies a flat pinned host
|
| 610 |
+
buffer straight into a flat device buffer on a private copy stream (async Host->Device only).
|
| 611 |
+
|
| 612 |
+
This is the seam that isolates all CUDA stream/event primitives from the residency logic. The residency
|
| 613 |
+
side only ever calls ``submit`` / ``wait``; ``_StagedCopier`` (pageable master + worker thread) and a
|
| 614 |
+
future non-CUDA device shim plug in behind the same interface without the residency code changing. Keep
|
| 615 |
+
every ``torch.cuda.*`` transfer primitive inside copiers like this one.
|
| 616 |
+
"""
|
| 617 |
+
|
| 618 |
+
def __init__(self, device: torch.device, debug: bool = False):
|
| 619 |
+
self.device = device
|
| 620 |
+
self.debug = debug
|
| 621 |
+
self.copy_stream = torch.cuda.Stream(device=device)
|
| 622 |
+
self._events = {} # key -> cuda.Event recording H2D completion
|
| 623 |
+
self._xfer = {} # key -> (ev_a, ev_b) transfer-timing events (debug only)
|
| 624 |
+
|
| 625 |
+
def submit(self, key, dst_flat: torch.Tensor, src_flat: torch.Tensor, gate_event=None):
|
| 626 |
+
"""Begin the H2D copy of ``src_flat`` into ``dst_flat``, gated behind ``gate_event`` (compute done with dst)."""
|
| 627 |
+
ev_a = ev_b = None
|
| 628 |
+
with torch.cuda.stream(self.copy_stream):
|
| 629 |
+
if gate_event is not None:
|
| 630 |
+
self.copy_stream.wait_event(gate_event) # do not overwrite a buffer still used by compute
|
| 631 |
+
if self.debug:
|
| 632 |
+
ev_a = torch.cuda.Event(enable_timing=True)
|
| 633 |
+
ev_b = torch.cuda.Event(enable_timing=True)
|
| 634 |
+
ev_a.record(self.copy_stream)
|
| 635 |
+
dst_flat.copy_(src_flat, non_blocking=True) # src is pinned -> async H2D
|
| 636 |
+
if self.debug:
|
| 637 |
+
ev_b.record(self.copy_stream)
|
| 638 |
+
self._xfer[key] = (ev_a, ev_b)
|
| 639 |
+
self._events[key] = self.copy_stream.record_event()
|
| 640 |
+
|
| 641 |
+
def wait(self, key):
|
| 642 |
+
"""Return the H2D-completion event for ``key`` (None if nothing is in flight for it)."""
|
| 643 |
+
return self._events.get(key)
|
| 644 |
+
|
| 645 |
+
def pop_xfer_timing(self, key):
|
| 646 |
+
"""Pop the (start, end) transfer-timing events recorded for ``key`` (debug only)."""
|
| 647 |
+
return self._xfer.pop(key, None)
|
| 648 |
+
|
| 649 |
+
def sync(self):
|
| 650 |
+
self.copy_stream.synchronize()
|
| 651 |
+
|
| 652 |
+
def reset(self):
|
| 653 |
+
self._events.clear()
|
| 654 |
+
self._xfer.clear()
|
| 655 |
+
|
| 656 |
+
|
| 657 |
+
class _StagedCopier:
|
| 658 |
+
"""
|
| 659 |
+
Transfer engine for ``LoRAStreamOffloader`` when the masters are kept in *pageable* host memory: each
|
| 660 |
+
H2D is staged through a small pool of pinned buffers filled by a background worker thread. Same
|
| 661 |
+
``submit`` / ``wait`` interface as ``_DirectCopier``, so the residency code is identical.
|
| 662 |
+
|
| 663 |
+
Why: pinning every streaming master (what ``_DirectCopier`` needs) can cost tens of GB of non-pageable
|
| 664 |
+
"shared GPU memory" (notably on Windows). Here the masters stay pageable and only ``num_staging`` pinned
|
| 665 |
+
buffers exist. A pageable->device copy is, inside the CUDA driver, already a synchronous serial
|
| 666 |
+
pageable->pinned->device transfer; doing it ourselves on a worker thread lets the pageable->pinned
|
| 667 |
+
memcpy overlap the previous block's pinned->device DMA and -- crucially -- keeps it off the main thread
|
| 668 |
+
so compute kernel launches are never blocked. That is the whole speed difference versus the old
|
| 669 |
+
synchronous pageable path this replaces.
|
| 670 |
+
|
| 671 |
+
Hazards handled:
|
| 672 |
+
- a staging buffer is reused only after the H2D that consumed it has completed (one consume-event per
|
| 673 |
+
buffer; the worker waits on it before overwriting);
|
| 674 |
+
- ``wait`` blocks the caller until the worker has actually enqueued the H2D and recorded its
|
| 675 |
+
completion event, so the residency layer can ``current_stream().wait_event`` it as usual.
|
| 676 |
+
"""
|
| 677 |
+
|
| 678 |
+
def __init__(self, device: torch.device, num_staging: int = 2, debug: bool = False):
|
| 679 |
+
self.device = device
|
| 680 |
+
self.num_staging = max(1, num_staging)
|
| 681 |
+
self.debug = debug
|
| 682 |
+
self.copy_stream = torch.cuda.Stream(device=device)
|
| 683 |
+
self._pool = ThreadPoolExecutor(max_workers=1)
|
| 684 |
+
self._futures = {} # key -> Future -> (load_event, (ev_a, ev_b))
|
| 685 |
+
self._xfer = {} # key -> (ev_a, ev_b) transfer-timing events (debug only)
|
| 686 |
+
self._staging = None # list[pinned uint8 tensor] (lazily sized to the flat block)
|
| 687 |
+
self._staging_free = None # list[cuda.Event | None]: the H2D that last consumed each buffer
|
| 688 |
+
self._rr = 0 # round-robin staging index (touched only on the worker thread)
|
| 689 |
+
|
| 690 |
+
def _ensure_staging(self, nbytes: int):
|
| 691 |
+
if self._staging is None:
|
| 692 |
+
self._staging = [torch.empty(nbytes, dtype=torch.uint8).pin_memory(device=self.device) for _ in range(self.num_staging)]
|
| 693 |
+
self._staging_free = [None] * self.num_staging
|
| 694 |
+
|
| 695 |
+
def _run(self, dst_flat: torch.Tensor, src_flat: torch.Tensor, gate_event):
|
| 696 |
+
# runs on the worker thread: pageable->pinned memcpy, then enqueue the async pinned->device H2D
|
| 697 |
+
idx = self._rr
|
| 698 |
+
self._rr = (idx + 1) % self.num_staging
|
| 699 |
+
staging = self._staging[idx]
|
| 700 |
+
|
| 701 |
+
free_ev = self._staging_free[idx]
|
| 702 |
+
if free_ev is not None:
|
| 703 |
+
free_ev.synchronize() # CPU(worker) waits until the prior H2D released this buffer -- off main thread
|
| 704 |
+
|
| 705 |
+
staging.copy_(src_flat) # pageable -> pinned, plain CPU memcpy
|
| 706 |
+
|
| 707 |
+
ev_a = ev_b = None
|
| 708 |
+
with torch.cuda.stream(self.copy_stream):
|
| 709 |
+
if gate_event is not None:
|
| 710 |
+
self.copy_stream.wait_event(gate_event) # do not overwrite a dst still used by compute
|
| 711 |
+
if self.debug:
|
| 712 |
+
ev_a = torch.cuda.Event(enable_timing=True)
|
| 713 |
+
ev_b = torch.cuda.Event(enable_timing=True)
|
| 714 |
+
ev_a.record(self.copy_stream)
|
| 715 |
+
dst_flat.copy_(staging, non_blocking=True) # pinned -> device, async
|
| 716 |
+
if self.debug:
|
| 717 |
+
ev_b.record(self.copy_stream)
|
| 718 |
+
done = self.copy_stream.record_event() # H2D complete == this staging buffer is free again
|
| 719 |
+
self._staging_free[idx] = done
|
| 720 |
+
return done, (ev_a, ev_b)
|
| 721 |
+
|
| 722 |
+
def submit(self, key, dst_flat: torch.Tensor, src_flat: torch.Tensor, gate_event=None):
|
| 723 |
+
self._ensure_staging(src_flat.numel()) # flat uint8 -> numel == byte size
|
| 724 |
+
self._futures[key] = self._pool.submit(self._run, dst_flat, src_flat, gate_event)
|
| 725 |
+
|
| 726 |
+
def wait(self, key):
|
| 727 |
+
fut = self._futures.get(key)
|
| 728 |
+
if fut is None:
|
| 729 |
+
return None
|
| 730 |
+
load_event, xfer = fut.result() # block until the worker enqueued the H2D (tiny if it ran ahead)
|
| 731 |
+
if self.debug and xfer[0] is not None:
|
| 732 |
+
self._xfer[key] = xfer
|
| 733 |
+
return load_event
|
| 734 |
+
|
| 735 |
+
def pop_xfer_timing(self, key):
|
| 736 |
+
return self._xfer.pop(key, None)
|
| 737 |
+
|
| 738 |
+
def _flush(self):
|
| 739 |
+
for fut in list(self._futures.values()):
|
| 740 |
+
fut.result()
|
| 741 |
+
|
| 742 |
+
def sync(self):
|
| 743 |
+
self._flush()
|
| 744 |
+
self.copy_stream.synchronize()
|
| 745 |
+
|
| 746 |
+
def reset(self):
|
| 747 |
+
self._flush()
|
| 748 |
+
self.copy_stream.synchronize() # ensure all in-flight H2D are done before clearing staging state
|
| 749 |
+
self._futures.clear()
|
| 750 |
+
self._xfer.clear()
|
| 751 |
+
if self._staging is not None:
|
| 752 |
+
self._staging_free = [None] * self.num_staging
|
| 753 |
+
|
| 754 |
+
def __del__(self):
|
| 755 |
+
pool = getattr(self, "_pool", None)
|
| 756 |
+
if pool is not None:
|
| 757 |
+
pool.shutdown(wait=False)
|
| 758 |
+
|
| 759 |
+
|
| 760 |
+
class LoRAStreamOffloader:
|
| 761 |
+
"""
|
| 762 |
+
H2D-only offloader for training where the base weights are frozen (e.g. LoRA / LoHa / LoKr).
|
| 763 |
+
|
| 764 |
+
The classic block swap (``ModelOffloader``) *exchanges* a block: it copies one block back to the
|
| 765 |
+
CPU (Device->Host) while copying the next one in (Host->Device). When the base weights never change,
|
| 766 |
+
the CPU already holds an identical copy, so the D2H half is pure overhead. This offloader keeps a
|
| 767 |
+
permanent (pinned) master copy of every streamed weight on the CPU and only ever copies Host->Device,
|
| 768 |
+
removing the D2H transfer and the CPU-side staging memcpy entirely.
|
| 769 |
+
|
| 770 |
+
Layout (N = num_blocks, S = blocks_to_swap, B = ring_size):
|
| 771 |
+
- ``N - S`` blocks stay resident on the GPU permanently.
|
| 772 |
+
- ``S`` "streaming" blocks are placed at even intervals; their ``Linear`` weights live on the CPU
|
| 773 |
+
(pinned) as masters and are streamed into a small ring of ``B`` GPU buffers (default 2: one being
|
| 774 |
+
computed on, one being prefetched). Non-Linear params / buffers of streaming blocks stay resident.
|
| 775 |
+
|
| 776 |
+
Mechanics:
|
| 777 |
+
- Because the streamed weights are frozen (no ``.grad`` to preserve), we swap the whole ``module.weight``
|
| 778 |
+
*reference* between its CPU master Parameter and a preallocated GPU ring Parameter, instead of the
|
| 779 |
+
``.data`` storage trick used by ``ModelOffloader``. No per-step ``cudaMalloc``.
|
| 780 |
+
- What is streamed is decided by ``swap_tensor_selector`` (default: every ``*Linear`` ``weight``). A
|
| 781 |
+
selector may return several tensors per module -- e.g. a quantized weight together with its scale
|
| 782 |
+
buffers -- as ``(module, attribute_name)`` pairs; parameters are rebound as ``nn.Parameter`` ring
|
| 783 |
+
views, registered buffers as plain tensor views.
|
| 784 |
+
- All swap weights of a block are packed into a single contiguous (byte) buffer -- one pinned CPU
|
| 785 |
+
buffer per streaming block and one GPU buffer per ring slot -- and the individual weights are
|
| 786 |
+
dtype/shape views into it. Loading a block is therefore a single H2D ``copy_`` instead of one
|
| 787 |
+
per Linear, which removes the per-tensor launch/driver overhead on PCIe/CPU-bound systems.
|
| 788 |
+
- Eviction is free: repoint ``module.weight`` back to the CPU master (no transfer). A ring slot is
|
| 789 |
+
overwritten only when its block is consumed, so the last ``B`` streamed blocks of a pass stay
|
| 790 |
+
resident. On the reverse pass they are exactly the first blocks needed, so the forward<->backward
|
| 791 |
+
turn-around needs no reload.
|
| 792 |
+
- Direction is implicit in the callback: ``submit_move_blocks_forward`` prefetches the block ``B``
|
| 793 |
+
streaming-slots *ahead* (larger index); the backward hook prefetches ``B`` slots *behind* (smaller
|
| 794 |
+
index). No explicit scheduler / no ``prepare_*_before_backward``.
|
| 795 |
+
- ``wait_for_block`` self-heals: if a needed streaming block is not resident in its slot (cold start,
|
| 796 |
+
or an inference pass boundary), it is loaded on demand.
|
| 797 |
+
"""
|
| 798 |
+
|
| 799 |
+
def __init__(
|
| 800 |
+
self,
|
| 801 |
+
block_type: str,
|
| 802 |
+
blocks: list[nn.Module],
|
| 803 |
+
num_blocks: int,
|
| 804 |
+
blocks_to_swap: int,
|
| 805 |
+
supports_backward: bool,
|
| 806 |
+
device: torch.device,
|
| 807 |
+
ring_size: int = 2,
|
| 808 |
+
use_pinned_memory: bool = True,
|
| 809 |
+
debug: bool = False,
|
| 810 |
+
swap_tensor_selector=None,
|
| 811 |
+
):
|
| 812 |
+
self.block_type = block_type
|
| 813 |
+
self._blocks = blocks
|
| 814 |
+
self.num_blocks = num_blocks # N
|
| 815 |
+
self.blocks_to_swap = blocks_to_swap # requested S
|
| 816 |
+
self.device = device
|
| 817 |
+
self.use_pinned_memory = use_pinned_memory
|
| 818 |
+
self.supports_backward = supports_backward
|
| 819 |
+
self.forward_only = not supports_backward
|
| 820 |
+
self.swap_tensor_selector = swap_tensor_selector if swap_tensor_selector is not None else default_swap_tensor_selector
|
| 821 |
+
|
| 822 |
+
import os
|
| 823 |
+
|
| 824 |
+
if not debug:
|
| 825 |
+
debug = os.getenv("MUSUBI_TUNER_OFFLOADER_DEBUG", "0") == "1"
|
| 826 |
+
self.debug = debug
|
| 827 |
+
self.debug_interval = int(os.getenv("MUSUBI_TUNER_OFFLOADER_DEBUG_INTERVAL", "10")) # steps between prints
|
| 828 |
+
|
| 829 |
+
assert device.type == "cuda", "LoRAStreamOffloader currently supports CUDA only"
|
| 830 |
+
|
| 831 |
+
# ---- streaming placement: S evenly spaced block indices (midpoint formula -> distinct for S <= N) ----
|
| 832 |
+
stream_idx = sorted({((2 * i + 1) * num_blocks) // (2 * blocks_to_swap) for i in range(blocks_to_swap)})
|
| 833 |
+
self.stream_idx = stream_idx
|
| 834 |
+
self.S = len(stream_idx) # actual streaming count (>=1; dedup is a no-op unless S is close to N)
|
| 835 |
+
self.rank = {b: k for k, b in enumerate(stream_idx)} # block_idx -> position in stream_idx
|
| 836 |
+
self.is_stream = [b in self.rank for b in range(num_blocks)]
|
| 837 |
+
self.B = min(ring_size, self.S) # clamp ring to number of streaming blocks
|
| 838 |
+
|
| 839 |
+
# ---- finetuning guard: H2D-only must never lose weight updates ----
|
| 840 |
+
self._job_cache = {} # block_idx -> [(module, attr_name, is_param) per swap tensor]
|
| 841 |
+
for b in stream_idx:
|
| 842 |
+
for m, name, is_param in self._jobs(b):
|
| 843 |
+
assert not (is_param and getattr(m, name).requires_grad), (
|
| 844 |
+
"LoRAStreamOffloader requires frozen base weights (LoRA / no full fine-tune). "
|
| 845 |
+
f"Found a trainable swap tensor {name} in block {b}."
|
| 846 |
+
)
|
| 847 |
+
|
| 848 |
+
# ---- transfer engine: owns the copy stream and all H2D primitives ----
|
| 849 |
+
# pinned masters -> direct async H2D; pageable masters -> stage through a worker thread + pinned pool
|
| 850 |
+
# (low "shared GPU memory" footprint, e.g. on Windows). Both expose the same submit/wait interface.
|
| 851 |
+
if use_pinned_memory:
|
| 852 |
+
self.copier = _DirectCopier(device, debug=self.debug)
|
| 853 |
+
else:
|
| 854 |
+
self.copier = _StagedCopier(device, num_staging=self.B, debug=self.debug)
|
| 855 |
+
|
| 856 |
+
# ---- runtime state (GPU buffers allocated lazily in prepare_block_devices_before_forward) ----
|
| 857 |
+
self.cpu_master = {} # block_idx -> [CPU (pinned) Parameter / buffer tensor per swap tensor] (views into cpu_flat)
|
| 858 |
+
self.cpu_flat = {} # block_idx -> flat (pinned) uint8 CPU tensor backing the masters
|
| 859 |
+
self.ring_param = None # [slot] -> [GPU nn.Parameter / buffer tensor per swap tensor] (views into ring_flat)
|
| 860 |
+
self.ring_flat = None # [slot] -> flat uint8 GPU tensor backing the ring params
|
| 861 |
+
self._layout = None # ([byte offset per swap tensor], total bytes) shared by all streaming blocks
|
| 862 |
+
self.in_slot = [None] * self.B # slot -> block_idx currently bound to this slot (or None)
|
| 863 |
+
self.free_event = [None] * self.B # slot -> cuda.Event recording when compute finished using the slot
|
| 864 |
+
|
| 865 |
+
self._wait_ctx = "fwd" # context tag for wait/load timing ("fwd" while in the forward loop, "bwd" in hooks)
|
| 866 |
+
if self.debug:
|
| 867 |
+
self._dbg_reset()
|
| 868 |
+
|
| 869 |
+
# ---- backward hooks: only where there is work (prefetch this block, or wait the previous one) ----
|
| 870 |
+
if supports_backward:
|
| 871 |
+
self.remove_handles = []
|
| 872 |
+
for i, block in enumerate(blocks):
|
| 873 |
+
hook = self._create_backward_hook(i)
|
| 874 |
+
if hook is not None:
|
| 875 |
+
self.remove_handles.append(block.register_full_backward_hook(hook))
|
| 876 |
+
|
| 877 |
+
print(
|
| 878 |
+
f"LoRAStreamOffloader[{block_type}]: H2D-only block swap. "
|
| 879 |
+
f"{self.S} streaming / {num_blocks} blocks, ring={self.B}, pinned={use_pinned_memory}. "
|
| 880 |
+
f"streaming indices: {stream_idx}"
|
| 881 |
+
)
|
| 882 |
+
|
| 883 |
+
# ------------------------------------------------------------------ helpers
|
| 884 |
+
|
| 885 |
+
def _jobs(self, block_idx: int) -> list[tuple[nn.Module, str, bool]]:
|
| 886 |
+
"""Swap jobs of a block: ``(module, attr_name, is_param)`` per streamed tensor, cached."""
|
| 887 |
+
cached = self._job_cache.get(block_idx)
|
| 888 |
+
if cached is None:
|
| 889 |
+
cached = []
|
| 890 |
+
for m, name in self.swap_tensor_selector(self._blocks[block_idx]):
|
| 891 |
+
if name in m._parameters:
|
| 892 |
+
is_param = True
|
| 893 |
+
elif name in m._buffers:
|
| 894 |
+
is_param = False
|
| 895 |
+
else:
|
| 896 |
+
raise ValueError(
|
| 897 |
+
f"swap tensor selector returned {name!r}, which is neither a parameter nor a"
|
| 898 |
+
f" registered buffer of {m.__class__.__name__} (block {block_idx})"
|
| 899 |
+
)
|
| 900 |
+
cached.append((m, name, is_param))
|
| 901 |
+
self._job_cache[block_idx] = cached
|
| 902 |
+
return cached
|
| 903 |
+
|
| 904 |
+
def _bind(self, block_idx: int, tensors: list[torch.Tensor]):
|
| 905 |
+
# parameters arrive as nn.Parameter, buffers as plain tensors; setattr keeps them in the
|
| 906 |
+
# module's _parameters / _buffers dicts respectively (buffer names are already registered)
|
| 907 |
+
for (m, name, _is_param), t in zip(self._jobs(block_idx), tensors):
|
| 908 |
+
setattr(m, name, t)
|
| 909 |
+
|
| 910 |
+
@staticmethod
|
| 911 |
+
def _compute_layout(weights: list[torch.Tensor]) -> tuple[list[int], int]:
|
| 912 |
+
"""Byte offsets (aligned so ``view(dtype)`` is valid for any element size) and total size of the flat buffer."""
|
| 913 |
+
align = 256
|
| 914 |
+
offsets = []
|
| 915 |
+
total = 0
|
| 916 |
+
for w in weights:
|
| 917 |
+
total = (total + align - 1) // align * align
|
| 918 |
+
offsets.append(total)
|
| 919 |
+
total += w.numel() * w.element_size()
|
| 920 |
+
return offsets, total
|
| 921 |
+
|
| 922 |
+
@staticmethod
|
| 923 |
+
def _flat_views(flat: torch.Tensor, weights: list[torch.Tensor], layout: tuple[list[int], int]) -> list[torch.Tensor]:
|
| 924 |
+
"""dtype/shape views into a flat uint8 buffer, one per swap tensor, following ``layout``."""
|
| 925 |
+
offsets, _ = layout
|
| 926 |
+
return [flat[off : off + w.numel() * w.element_size()].view(w.dtype).view(w.shape) for off, w in zip(offsets, weights)]
|
| 927 |
+
|
| 928 |
+
def _load(self, rank: int, slot: int, ctx: str = "fwd"):
|
| 929 |
+
"""Stream the streaming block at ``rank`` into ring ``slot`` (H2D only), repointing weights."""
|
| 930 |
+
blk = self.stream_idx[rank]
|
| 931 |
+
|
| 932 |
+
# already resident (e.g. ring survived a re-prepare): the slot content is still identical to the
|
| 933 |
+
# immutable CPU master, so just rebind the weights to the ring -- no transfer needed
|
| 934 |
+
if self.in_slot[slot] == blk:
|
| 935 |
+
self._bind(blk, self.ring_param[slot])
|
| 936 |
+
return
|
| 937 |
+
|
| 938 |
+
# evict the slot's current owner: repoint its weights back to the CPU master (no transfer)
|
| 939 |
+
prev = self.in_slot[slot]
|
| 940 |
+
if prev is not None:
|
| 941 |
+
self._bind(prev, self.cpu_master[prev])
|
| 942 |
+
|
| 943 |
+
# single coalesced H2D copy of the whole block, gated behind the slot's free_event (compute done
|
| 944 |
+
# with it). The ring params are views into ring_flat, so the copy bumps their autograd version
|
| 945 |
+
# counter: safe under gradient checkpointing (weights are re-read at recompute time, and a slot is
|
| 946 |
+
# only overwritten after its free_event); without checkpointing autograd raises a version error
|
| 947 |
+
# instead of silently using stale-saved weights -- which the old per-tensor `.data.copy_` hid.
|
| 948 |
+
self.copier.submit(blk, self.ring_flat[slot], self.cpu_flat[blk], self.free_event[slot])
|
| 949 |
+
|
| 950 |
+
self._bind(blk, self.ring_param[slot]) # reference swap (no .data trick, no cudaMalloc)
|
| 951 |
+
self.in_slot[slot] = blk
|
| 952 |
+
|
| 953 |
+
if self.debug:
|
| 954 |
+
c = self._cur[ctx]
|
| 955 |
+
c["loads"] += 1
|
| 956 |
+
xfer = self.copier.pop_xfer_timing(blk)
|
| 957 |
+
if xfer is not None:
|
| 958 |
+
c["xfer_ev"].append(xfer)
|
| 959 |
+
|
| 960 |
+
# ------------------------------------------------------------------ public interface (mirrors ModelOffloader)
|
| 961 |
+
|
| 962 |
+
def set_forward_only(self, forward_only: bool):
|
| 963 |
+
self.copier.sync()
|
| 964 |
+
self.forward_only = forward_only
|
| 965 |
+
|
| 966 |
+
def __del__(self):
|
| 967 |
+
if getattr(self, "supports_backward", False):
|
| 968 |
+
for handle in getattr(self, "remove_handles", []):
|
| 969 |
+
handle.remove()
|
| 970 |
+
|
| 971 |
+
def prepare_block_devices_before_forward(self, blocks: list[nn.Module]):
|
| 972 |
+
if self.S == 0:
|
| 973 |
+
return
|
| 974 |
+
|
| 975 |
+
if self.debug:
|
| 976 |
+
print(f"[{self.block_type}] Prepare block devices before forward (H2D-only)")
|
| 977 |
+
|
| 978 |
+
first_time = not self.cpu_master # device placement is only needed on the very first call
|
| 979 |
+
cpu_device = torch.device("cpu")
|
| 980 |
+
for i, b in enumerate(blocks):
|
| 981 |
+
if not self.is_stream[i]:
|
| 982 |
+
if first_time:
|
| 983 |
+
b.to(self.device) # resident block: buffers + Linear weights on GPU permanently
|
| 984 |
+
weighs_to_device(b, self.device)
|
| 985 |
+
continue
|
| 986 |
+
|
| 987 |
+
if not first_time:
|
| 988 |
+
# re-prepare (e.g. around in-training sampling): the non-swap parts are already on the device
|
| 989 |
+
# and the CPU masters have stayed on the CPU throughout, so just repoint weights to the masters.
|
| 990 |
+
# IMPORTANT: do NOT call b.to(device) here -- it would drag the CPU masters onto the GPU and
|
| 991 |
+
# they would never be released (every swap block would end up resident).
|
| 992 |
+
self._bind(i, self.cpu_master[i])
|
| 993 |
+
continue
|
| 994 |
+
|
| 995 |
+
jobs = self._jobs(i)
|
| 996 |
+
tensors = [getattr(m, name).data for m, name, _ in jobs]
|
| 997 |
+
if self._layout is None:
|
| 998 |
+
self._layout = self._compute_layout(tensors) # first streaming block defines the shared layout
|
| 999 |
+
flat = torch.empty(self._layout[1], dtype=torch.uint8, device=cpu_device)
|
| 1000 |
+
if self.use_pinned_memory:
|
| 1001 |
+
flat = flat.pin_memory(device=self.device)
|
| 1002 |
+
views = self._flat_views(flat, tensors, self._layout)
|
| 1003 |
+
master = []
|
| 1004 |
+
|
| 1005 |
+
if all(t.device.type == "cpu" for t in tensors):
|
| 1006 |
+
# CPU-direct: build the flat masters host-to-host, detach the swap tensors from the
|
| 1007 |
+
# module so the block's device move cannot drag them, then restore them as the masters.
|
| 1008 |
+
# Avoids the H2D+D2H round trip of every streamed tensor that the general path below
|
| 1009 |
+
# pays when the model starts on the CPU.
|
| 1010 |
+
for (m, name, is_param), t, view in zip(jobs, tensors, views):
|
| 1011 |
+
view.copy_(t) # host-to-host memcpy into the flat master
|
| 1012 |
+
placeholder = torch.empty(0, dtype=t.dtype, device=cpu_device)
|
| 1013 |
+
if is_param:
|
| 1014 |
+
m._parameters[name].data = placeholder
|
| 1015 |
+
else:
|
| 1016 |
+
m._buffers[name] = placeholder
|
| 1017 |
+
b.to(self.device) # non-swap params/buffers to the device; the placeholders are empty
|
| 1018 |
+
for (m, name, is_param), view in zip(jobs, views):
|
| 1019 |
+
if is_param:
|
| 1020 |
+
p = m._parameters[name]
|
| 1021 |
+
p.data = view
|
| 1022 |
+
master.append(p) # keep the original Parameter object as the persistent master
|
| 1023 |
+
else:
|
| 1024 |
+
m._buffers[name] = view
|
| 1025 |
+
master.append(view)
|
| 1026 |
+
else:
|
| 1027 |
+
# swap tensors already live on an accelerator: move the whole block to the device
|
| 1028 |
+
# (buffers, norms, bias), then pull the swap tensors back into the flat master (one-time D2H)
|
| 1029 |
+
b.to(self.device)
|
| 1030 |
+
for (m, name, is_param), view in zip(jobs, views):
|
| 1031 |
+
t = getattr(m, name)
|
| 1032 |
+
view.copy_(t.data)
|
| 1033 |
+
if is_param:
|
| 1034 |
+
t.data = view
|
| 1035 |
+
master.append(t) # keep the original Parameter object as the persistent master
|
| 1036 |
+
else:
|
| 1037 |
+
m._buffers[name] = view
|
| 1038 |
+
master.append(view)
|
| 1039 |
+
|
| 1040 |
+
self.cpu_flat[i] = flat
|
| 1041 |
+
self.cpu_master[i] = master
|
| 1042 |
+
|
| 1043 |
+
# validate homogeneous streaming blocks (shared ring template)
|
| 1044 |
+
template = self.cpu_master[self.stream_idx[0]]
|
| 1045 |
+
for b in self.stream_idx[1:]:
|
| 1046 |
+
assert len(self.cpu_master[b]) == len(template), f"block {b} has a different number of swap tensors"
|
| 1047 |
+
for p, t in zip(self.cpu_master[b], template):
|
| 1048 |
+
assert p.data.shape == t.data.shape and p.data.dtype == t.data.dtype, (
|
| 1049 |
+
f"block {b} swap-tensor shape/dtype differs from the streaming template"
|
| 1050 |
+
)
|
| 1051 |
+
|
| 1052 |
+
# preallocate the GPU ring (once); copies happen into these flat buffers, never reallocated
|
| 1053 |
+
if self.ring_param is None:
|
| 1054 |
+
template_jobs = self._jobs(self.stream_idx[0])
|
| 1055 |
+
template_weights = [p.data for p in template]
|
| 1056 |
+
self.ring_flat = [torch.empty(self._layout[1], dtype=torch.uint8, device=self.device) for _ in range(self.B)]
|
| 1057 |
+
self.ring_param = [
|
| 1058 |
+
[
|
| 1059 |
+
nn.Parameter(view, requires_grad=False) if is_param else view
|
| 1060 |
+
for (_m, _name, is_param), view in zip(template_jobs, self._flat_views(flat, template_weights, self._layout))
|
| 1061 |
+
]
|
| 1062 |
+
for flat in self.ring_flat
|
| 1063 |
+
]
|
| 1064 |
+
|
| 1065 |
+
# reset transfer state and preload the first B streaming blocks. in_slot deliberately survives:
|
| 1066 |
+
# after a completed backward pass the ring already holds ranks 0..B-1, so these loads become
|
| 1067 |
+
# no-op rebinds (see _load) instead of redundant cold-start transfers.
|
| 1068 |
+
self.free_event = [None] * self.B
|
| 1069 |
+
self.copier.reset()
|
| 1070 |
+
for k in range(self.B):
|
| 1071 |
+
self._load(k, k)
|
| 1072 |
+
|
| 1073 |
+
_synchronize_device(self.device)
|
| 1074 |
+
_clean_memory_on_device(self.device)
|
| 1075 |
+
|
| 1076 |
+
def wait_for_block(self, block_idx: int):
|
| 1077 |
+
if self.S == 0 or not self.is_stream[block_idx]:
|
| 1078 |
+
return
|
| 1079 |
+
ctx = self._wait_ctx
|
| 1080 |
+
if self.debug and ctx == "fwd" and block_idx == self.stream_idx[0]:
|
| 1081 |
+
self._dbg_boundary() # a forward pass is starting -> finalize the previous step's stats
|
| 1082 |
+
j = self.rank[block_idx]
|
| 1083 |
+
slot = j % self.B
|
| 1084 |
+
if self.in_slot[slot] != block_idx:
|
| 1085 |
+
# self-healing: not resident (cold start / inference pass boundary / same-direction restart)
|
| 1086 |
+
if self.debug:
|
| 1087 |
+
self._cur[ctx]["self_heal"] += 1
|
| 1088 |
+
self._load(j, slot, ctx)
|
| 1089 |
+
ev = self.copier.wait(block_idx)
|
| 1090 |
+
if ev is None:
|
| 1091 |
+
return
|
| 1092 |
+
if not self.debug:
|
| 1093 |
+
torch.cuda.current_stream().wait_event(ev) # compute waits for H2D (GPU-side, does not block CPU)
|
| 1094 |
+
return
|
| 1095 |
+
# debug: measure the actual GPU stall (idle time the compute stream spends on wait_event)
|
| 1096 |
+
s = torch.cuda.current_stream()
|
| 1097 |
+
c = self._cur[ctx]
|
| 1098 |
+
c["waits"] += 1
|
| 1099 |
+
if not ev.query():
|
| 1100 |
+
c["not_ready"] += 1 # H2D not finished when compute reached it -> a real stall (cheap, no sync)
|
| 1101 |
+
ev_a = torch.cuda.Event(enable_timing=True)
|
| 1102 |
+
ev_b = torch.cuda.Event(enable_timing=True)
|
| 1103 |
+
ev_a.record(s)
|
| 1104 |
+
s.wait_event(ev)
|
| 1105 |
+
ev_b.record(s)
|
| 1106 |
+
c["stall_ev"].append((ev_a, ev_b))
|
| 1107 |
+
# staged copier records its transfer-timing events on the worker thread, so they only become
|
| 1108 |
+
# available once wait() has joined it; the direct copier already popped them in _load (returns None)
|
| 1109 |
+
xfer = self.copier.pop_xfer_timing(block_idx)
|
| 1110 |
+
if xfer is not None:
|
| 1111 |
+
c["xfer_ev"].append(xfer)
|
| 1112 |
+
|
| 1113 |
+
def submit_move_blocks_forward(self, blocks: list[nn.Module], block_idx: int):
|
| 1114 |
+
# consumed block_idx in the forward pass: free its slot and prefetch the block B streaming-slots ahead
|
| 1115 |
+
if self.S == 0 or not self.is_stream[block_idx]:
|
| 1116 |
+
return
|
| 1117 |
+
j = self.rank[block_idx]
|
| 1118 |
+
self.free_event[j % self.B] = torch.cuda.current_stream().record_event()
|
| 1119 |
+
if j + self.B < self.S:
|
| 1120 |
+
self._load(j + self.B, (j + self.B) % self.B, "fwd") # same slot as rank j; evicts rank j
|
| 1121 |
+
elif self.forward_only and j == self.S - 1 and self.S > self.B:
|
| 1122 |
+
# inference: no backward pass will consume the retained tail, and the next forward pass starts
|
| 1123 |
+
# from rank 0 again. Wrap around: all tail slots are free once the last streaming block is
|
| 1124 |
+
# consumed, so preload the first B ranks now. The H2D overlaps with the remaining (resident)
|
| 1125 |
+
# blocks and the output layers, instead of stalling at the start of the next pass (self-heal).
|
| 1126 |
+
for k in range(self.B):
|
| 1127 |
+
self._load(k, k, "fwd")
|
| 1128 |
+
|
| 1129 |
+
def _create_backward_hook(self, block_index: int):
|
| 1130 |
+
prefetch = self.is_stream[block_index]
|
| 1131 |
+
wait_prev = block_index - 1 >= 0 and self.is_stream[block_index - 1]
|
| 1132 |
+
if not prefetch and not wait_prev:
|
| 1133 |
+
return None
|
| 1134 |
+
|
| 1135 |
+
def backward_hook(module, grad_input, grad_output):
|
| 1136 |
+
if prefetch:
|
| 1137 |
+
# consumed block_index in backward: free its slot and prefetch B streaming-slots behind
|
| 1138 |
+
j = self.rank[block_index]
|
| 1139 |
+
self.free_event[j % self.B] = torch.cuda.current_stream().record_event()
|
| 1140 |
+
if j - self.B >= 0:
|
| 1141 |
+
self._load(j - self.B, (j - self.B) % self.B, "bwd") # same slot as rank j; evicts rank j
|
| 1142 |
+
if wait_prev:
|
| 1143 |
+
self._wait_ctx = "bwd"
|
| 1144 |
+
self.wait_for_block(block_index - 1) # ensure the next block to run is ready
|
| 1145 |
+
self._wait_ctx = "fwd"
|
| 1146 |
+
return None
|
| 1147 |
+
|
| 1148 |
+
return backward_hook
|
| 1149 |
+
|
| 1150 |
+
# ------------------------------------------------------------------ debug timing
|
| 1151 |
+
|
| 1152 |
+
def _dbg_reset(self):
|
| 1153 |
+
self._dbg_empty = lambda: {"stall_ev": [], "xfer_ev": [], "waits": 0, "not_ready": 0, "self_heal": 0, "loads": 0}
|
| 1154 |
+
self._cur = {"fwd": self._dbg_empty(), "bwd": self._dbg_empty()}
|
| 1155 |
+
self._roll = {"fwd": defaultdict(float), "bwd": defaultdict(float)}
|
| 1156 |
+
self._step = 0
|
| 1157 |
+
|
| 1158 |
+
def _dbg_boundary(self):
|
| 1159 |
+
# finalize the events accumulated for the step that just ended, then print every debug_interval steps
|
| 1160 |
+
if not any(self._cur[c]["waits"] or self._cur[c]["loads"] for c in ("fwd", "bwd")):
|
| 1161 |
+
return # very first forward: nothing finished yet
|
| 1162 |
+
torch.cuda.synchronize() # ensure timing events are complete (debug only)
|
| 1163 |
+
for ctx in ("fwd", "bwd"):
|
| 1164 |
+
c, r = self._cur[ctx], self._roll[ctx]
|
| 1165 |
+
r["stall_ms"] += sum(a.elapsed_time(b) for a, b in c["stall_ev"])
|
| 1166 |
+
r["xfer_ms"] += sum(a.elapsed_time(b) for a, b in c["xfer_ev"])
|
| 1167 |
+
for k in ("waits", "not_ready", "self_heal", "loads"):
|
| 1168 |
+
r[k] += c[k]
|
| 1169 |
+
self._cur[ctx] = self._dbg_empty()
|
| 1170 |
+
self._step += 1
|
| 1171 |
+
if self._step % self.debug_interval == 0:
|
| 1172 |
+
self._dbg_print()
|
| 1173 |
+
|
| 1174 |
+
def _dbg_print(self):
|
| 1175 |
+
n = self.debug_interval
|
| 1176 |
+
print(f"[{self.block_type}] H2D-only timing (avg over {n} steps):")
|
| 1177 |
+
for ctx in ("fwd", "bwd"):
|
| 1178 |
+
r = self._roll[ctx]
|
| 1179 |
+
print(
|
| 1180 |
+
f" {ctx}: stall {r['stall_ms'] / n:6.2f} ms/step, H2D {r['xfer_ms'] / n:6.2f} ms/step"
|
| 1181 |
+
f" | waits {r['waits'] / n:.1f}, not-ready {r['not_ready'] / n:.1f},"
|
| 1182 |
+
f" self-heal {r['self_heal'] / n:.1f}, loads {r['loads'] / n:.1f}"
|
| 1183 |
+
)
|
| 1184 |
+
self._roll[ctx] = defaultdict(float)
|
third_party/musubi_tuner/modules/fp8_optimization_utils.py
ADDED
|
@@ -0,0 +1,565 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
from typing import List, Optional, Union
|
| 3 |
+
import torch
|
| 4 |
+
import torch.nn as nn
|
| 5 |
+
import torch.nn.functional as F
|
| 6 |
+
|
| 7 |
+
import logging
|
| 8 |
+
|
| 9 |
+
from tqdm import tqdm
|
| 10 |
+
|
| 11 |
+
from musubi_tuner.utils.safetensors_utils import MemoryEfficientSafeOpen, TensorWeightAdapter, WeightTransformHooks
|
| 12 |
+
|
| 13 |
+
logger = logging.getLogger(__name__)
|
| 14 |
+
logging.basicConfig(level=logging.INFO)
|
| 15 |
+
|
| 16 |
+
from musubi_tuner.utils.device_utils import clean_memory_on_device
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def calculate_fp8_maxval(exp_bits=4, mantissa_bits=3, sign_bits=1):
|
| 20 |
+
"""
|
| 21 |
+
Calculate the maximum representable value in FP8 format.
|
| 22 |
+
Default is E4M3 format (4-bit exponent, 3-bit mantissa, 1-bit sign). Only supports E4M3 and E5M2 with sign bit.
|
| 23 |
+
|
| 24 |
+
Args:
|
| 25 |
+
exp_bits (int): Number of exponent bits
|
| 26 |
+
mantissa_bits (int): Number of mantissa bits
|
| 27 |
+
sign_bits (int): Number of sign bits (0 or 1)
|
| 28 |
+
|
| 29 |
+
Returns:
|
| 30 |
+
float: Maximum value representable in FP8 format
|
| 31 |
+
"""
|
| 32 |
+
assert exp_bits + mantissa_bits + sign_bits == 8, "Total bits must be 8"
|
| 33 |
+
if exp_bits == 4 and mantissa_bits == 3 and sign_bits == 1:
|
| 34 |
+
return torch.finfo(torch.float8_e4m3fn).max
|
| 35 |
+
elif exp_bits == 5 and mantissa_bits == 2 and sign_bits == 1:
|
| 36 |
+
return torch.finfo(torch.float8_e5m2).max
|
| 37 |
+
else:
|
| 38 |
+
raise ValueError(f"Unsupported FP8 format: E{exp_bits}M{mantissa_bits} with sign_bits={sign_bits}")
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
# The following is a manual calculation method (wrong implementation for E5M2), kept for reference.
|
| 42 |
+
"""
|
| 43 |
+
# Calculate exponent bias
|
| 44 |
+
bias = 2 ** (exp_bits - 1) - 1
|
| 45 |
+
|
| 46 |
+
# Calculate maximum mantissa value
|
| 47 |
+
mantissa_max = 1.0
|
| 48 |
+
for i in range(mantissa_bits - 1):
|
| 49 |
+
mantissa_max += 2 ** -(i + 1)
|
| 50 |
+
|
| 51 |
+
# Calculate maximum value
|
| 52 |
+
max_value = mantissa_max * (2 ** (2**exp_bits - 1 - bias))
|
| 53 |
+
|
| 54 |
+
return max_value
|
| 55 |
+
"""
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def quantize_fp8(tensor, scale, fp8_dtype, max_value, min_value):
|
| 59 |
+
"""
|
| 60 |
+
Quantize a tensor to FP8 format using PyTorch's native FP8 dtype support.
|
| 61 |
+
|
| 62 |
+
Args:
|
| 63 |
+
tensor (torch.Tensor): Tensor to quantize
|
| 64 |
+
scale (float or torch.Tensor): Scale factor
|
| 65 |
+
fp8_dtype (torch.dtype): Target FP8 dtype (torch.float8_e4m3fn or torch.float8_e5m2)
|
| 66 |
+
max_value (float): Maximum representable value in FP8
|
| 67 |
+
min_value (float): Minimum representable value in FP8
|
| 68 |
+
|
| 69 |
+
Returns:
|
| 70 |
+
torch.Tensor: Quantized tensor in FP8 format
|
| 71 |
+
"""
|
| 72 |
+
tensor = tensor.to(torch.float32) # ensure tensor is in float32 for division
|
| 73 |
+
|
| 74 |
+
# Create scaled tensor
|
| 75 |
+
tensor = torch.div(tensor, scale).nan_to_num_(0.0) # handle NaN values, equivalent to nonzero_mask in previous function
|
| 76 |
+
|
| 77 |
+
# Clamp tensor to range
|
| 78 |
+
tensor = tensor.clamp_(min=min_value, max=max_value)
|
| 79 |
+
|
| 80 |
+
# Convert to FP8 dtype
|
| 81 |
+
tensor = tensor.to(fp8_dtype)
|
| 82 |
+
|
| 83 |
+
return tensor
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
def optimize_state_dict_with_fp8(
|
| 87 |
+
state_dict: dict,
|
| 88 |
+
calc_device: Union[str, torch.device],
|
| 89 |
+
target_layer_keys: Optional[list[str]] = None,
|
| 90 |
+
exclude_layer_keys: Optional[list[str]] = None,
|
| 91 |
+
exp_bits: int = 4,
|
| 92 |
+
mantissa_bits: int = 3,
|
| 93 |
+
move_to_device: bool = False,
|
| 94 |
+
quantization_mode: str = "block",
|
| 95 |
+
block_size: Optional[int] = 64,
|
| 96 |
+
):
|
| 97 |
+
"""
|
| 98 |
+
Optimize Linear layer weights in a model's state dict to FP8 format. The state dict is modified in-place.
|
| 99 |
+
This function is a static version of load_safetensors_with_fp8_optimization without loading from files.
|
| 100 |
+
|
| 101 |
+
Args:
|
| 102 |
+
state_dict (dict): State dict to optimize, replaced in-place
|
| 103 |
+
calc_device (str): Device to quantize tensors on
|
| 104 |
+
target_layer_keys (list, optional): Layer key patterns to target (None for all Linear layers)
|
| 105 |
+
exclude_layer_keys (list, optional): Layer key patterns to exclude
|
| 106 |
+
exp_bits (int): Number of exponent bits
|
| 107 |
+
mantissa_bits (int): Number of mantissa bits
|
| 108 |
+
move_to_device (bool): Move optimized tensors to the calculating device
|
| 109 |
+
|
| 110 |
+
Returns:
|
| 111 |
+
dict: FP8 optimized state dict
|
| 112 |
+
"""
|
| 113 |
+
if exp_bits == 4 and mantissa_bits == 3:
|
| 114 |
+
fp8_dtype = torch.float8_e4m3fn
|
| 115 |
+
elif exp_bits == 5 and mantissa_bits == 2:
|
| 116 |
+
fp8_dtype = torch.float8_e5m2
|
| 117 |
+
else:
|
| 118 |
+
raise ValueError(f"Unsupported FP8 format: E{exp_bits}M{mantissa_bits}")
|
| 119 |
+
|
| 120 |
+
# Calculate FP8 max value
|
| 121 |
+
max_value = calculate_fp8_maxval(exp_bits, mantissa_bits)
|
| 122 |
+
min_value = -max_value # this function supports only signed FP8
|
| 123 |
+
|
| 124 |
+
# Create optimized state dict
|
| 125 |
+
optimized_count = 0
|
| 126 |
+
|
| 127 |
+
# Enumerate tarket keys
|
| 128 |
+
target_state_dict_keys = []
|
| 129 |
+
for key in state_dict.keys():
|
| 130 |
+
# Check if it's a weight key and matches target patterns
|
| 131 |
+
is_target = (target_layer_keys is None or any(pattern in key for pattern in target_layer_keys)) and key.endswith(".weight")
|
| 132 |
+
is_excluded = exclude_layer_keys is not None and any(pattern in key for pattern in exclude_layer_keys)
|
| 133 |
+
is_target = is_target and not is_excluded
|
| 134 |
+
|
| 135 |
+
if is_target and isinstance(state_dict[key], torch.Tensor):
|
| 136 |
+
target_state_dict_keys.append(key)
|
| 137 |
+
|
| 138 |
+
# Process each key
|
| 139 |
+
for key in tqdm(target_state_dict_keys):
|
| 140 |
+
value = state_dict[key]
|
| 141 |
+
|
| 142 |
+
# Save original device and dtype
|
| 143 |
+
original_device = value.device
|
| 144 |
+
original_dtype = value.dtype
|
| 145 |
+
|
| 146 |
+
# Move to calculation device
|
| 147 |
+
if calc_device is not None:
|
| 148 |
+
value = value.to(calc_device)
|
| 149 |
+
|
| 150 |
+
quantized_weight, scale_tensor = quantize_weight(key, value, fp8_dtype, max_value, min_value, quantization_mode, block_size)
|
| 151 |
+
|
| 152 |
+
# Add to state dict using original key for weight and new key for scale
|
| 153 |
+
fp8_key = key # Maintain original key
|
| 154 |
+
scale_key = key.replace(".weight", ".scale_weight")
|
| 155 |
+
|
| 156 |
+
if not move_to_device:
|
| 157 |
+
quantized_weight = quantized_weight.to(original_device)
|
| 158 |
+
|
| 159 |
+
# keep scale shape: [1] or [out,1] or [out, num_blocks, 1]. We can determine the quantization mode from the shape of scale_weight in the patched model.
|
| 160 |
+
scale_tensor = scale_tensor.to(dtype=original_dtype, device=quantized_weight.device)
|
| 161 |
+
|
| 162 |
+
state_dict[fp8_key] = quantized_weight
|
| 163 |
+
state_dict[scale_key] = scale_tensor
|
| 164 |
+
|
| 165 |
+
optimized_count += 1
|
| 166 |
+
|
| 167 |
+
if calc_device is not None: # optimized_count % 10 == 0 and
|
| 168 |
+
# free memory on calculation device
|
| 169 |
+
clean_memory_on_device(calc_device)
|
| 170 |
+
|
| 171 |
+
logger.info(f"Number of optimized Linear layers: {optimized_count}")
|
| 172 |
+
return state_dict
|
| 173 |
+
|
| 174 |
+
|
| 175 |
+
def quantize_weight(
|
| 176 |
+
key: str,
|
| 177 |
+
tensor: torch.Tensor,
|
| 178 |
+
fp8_dtype: torch.dtype,
|
| 179 |
+
max_value: float,
|
| 180 |
+
min_value: float,
|
| 181 |
+
quantization_mode: str = "block",
|
| 182 |
+
block_size: int = 64,
|
| 183 |
+
):
|
| 184 |
+
original_shape = tensor.shape
|
| 185 |
+
|
| 186 |
+
# Determine quantization mode
|
| 187 |
+
if quantization_mode == "block":
|
| 188 |
+
if tensor.ndim != 2:
|
| 189 |
+
quantization_mode = "tensor" # fallback to per-tensor
|
| 190 |
+
else:
|
| 191 |
+
out_features, in_features = tensor.shape
|
| 192 |
+
if in_features % block_size != 0:
|
| 193 |
+
quantization_mode = "channel" # fallback to per-channel
|
| 194 |
+
logger.warning(
|
| 195 |
+
f"Layer {key} with shape {tensor.shape} is not divisible by block_size {block_size}, fallback to per-channel quantization."
|
| 196 |
+
)
|
| 197 |
+
else:
|
| 198 |
+
num_blocks = in_features // block_size
|
| 199 |
+
tensor = tensor.contiguous().view(out_features, num_blocks, block_size) # [out, num_blocks, block_size]
|
| 200 |
+
elif quantization_mode == "channel":
|
| 201 |
+
if tensor.ndim != 2:
|
| 202 |
+
quantization_mode = "tensor" # fallback to per-tensor
|
| 203 |
+
|
| 204 |
+
# Calculate scale factor (per-tensor or per-output-channel with percentile or max)
|
| 205 |
+
# value shape is expected to be [out_features, in_features] for Linear weights
|
| 206 |
+
if quantization_mode == "channel" or quantization_mode == "block":
|
| 207 |
+
# row-wise percentile to avoid being dominated by outliers
|
| 208 |
+
# result shape: [out_features, 1] or [out_features, num_blocks, 1]
|
| 209 |
+
scale_dim = 1 if quantization_mode == "channel" else 2
|
| 210 |
+
abs_w = torch.abs(tensor)
|
| 211 |
+
|
| 212 |
+
# shape: [out_features, 1] or [out_features, num_blocks, 1]
|
| 213 |
+
row_max = torch.max(abs_w, dim=scale_dim, keepdim=True).values
|
| 214 |
+
scale = row_max / max_value
|
| 215 |
+
|
| 216 |
+
else:
|
| 217 |
+
# per-tensor
|
| 218 |
+
tensor_max = torch.max(torch.abs(tensor).view(-1))
|
| 219 |
+
scale = tensor_max / max_value
|
| 220 |
+
|
| 221 |
+
# print(f"Optimizing {key} with scale: {scale}")
|
| 222 |
+
|
| 223 |
+
# numerical safety
|
| 224 |
+
scale = torch.clamp(scale, min=1e-8)
|
| 225 |
+
scale = scale.to(torch.float32) # ensure scale is in float32 for division
|
| 226 |
+
|
| 227 |
+
# Quantize weight to FP8 (scale can be scalar or [out,1], broadcasting works)
|
| 228 |
+
quantized_weight = quantize_fp8(tensor, scale, fp8_dtype, max_value, min_value)
|
| 229 |
+
|
| 230 |
+
# If block-wise, restore original shape
|
| 231 |
+
if quantization_mode == "block":
|
| 232 |
+
quantized_weight = quantized_weight.view(original_shape) # restore to original shape [out, in]
|
| 233 |
+
|
| 234 |
+
return quantized_weight, scale
|
| 235 |
+
|
| 236 |
+
|
| 237 |
+
def load_safetensors_with_fp8_optimization(
|
| 238 |
+
model_files: List[str],
|
| 239 |
+
calc_device: Union[str, torch.device],
|
| 240 |
+
target_layer_keys=None,
|
| 241 |
+
exclude_layer_keys=None,
|
| 242 |
+
exp_bits=4,
|
| 243 |
+
mantissa_bits=3,
|
| 244 |
+
move_to_device=False,
|
| 245 |
+
weight_hook=None,
|
| 246 |
+
quantization_mode: str = "block",
|
| 247 |
+
block_size: Optional[int] = 64,
|
| 248 |
+
disable_numpy_memmap: bool = False,
|
| 249 |
+
weight_transform_hooks: Optional[WeightTransformHooks] = None,
|
| 250 |
+
allow_prequantized_fp8: bool = False,
|
| 251 |
+
) -> dict:
|
| 252 |
+
"""
|
| 253 |
+
Load weight tensors from safetensors files and merge LoRA weights into the state dict with explicit FP8 optimization.
|
| 254 |
+
|
| 255 |
+
Args:
|
| 256 |
+
model_files (list[str]): List of model files to load
|
| 257 |
+
calc_device (str or torch.device): Device to quantize tensors on
|
| 258 |
+
target_layer_keys (list, optional): Layer key patterns to target for optimization (None for all Linear layers)
|
| 259 |
+
exclude_layer_keys (list, optional): Layer key patterns to exclude from optimization
|
| 260 |
+
exp_bits (int): Number of exponent bits
|
| 261 |
+
mantissa_bits (int): Number of mantissa bits
|
| 262 |
+
move_to_device (bool): Move optimized tensors to the calculating device
|
| 263 |
+
weight_hook (callable, optional): Function to apply to each weight tensor before optimization
|
| 264 |
+
quantization_mode (str): Quantization mode, "tensor", "channel", or "block"
|
| 265 |
+
block_size (int, optional): Block size for block-wise quantization (used if quantization_mode is "block")
|
| 266 |
+
disable_numpy_memmap (bool): Disable numpy memmap when loading safetensors
|
| 267 |
+
weight_transform_hooks (WeightTransformHooks, optional): Hooks for weight transformation during loading
|
| 268 |
+
allow_prequantized_fp8 (bool): If True, target weights that are already FP8 are kept as-is instead of
|
| 269 |
+
raising; their scale is expected to be supplied separately (e.g. via weight_transform_hooks).
|
| 270 |
+
|
| 271 |
+
Returns:
|
| 272 |
+
dict: FP8 optimized state dict
|
| 273 |
+
"""
|
| 274 |
+
if exp_bits == 4 and mantissa_bits == 3:
|
| 275 |
+
fp8_dtype = torch.float8_e4m3fn
|
| 276 |
+
elif exp_bits == 5 and mantissa_bits == 2:
|
| 277 |
+
fp8_dtype = torch.float8_e5m2
|
| 278 |
+
else:
|
| 279 |
+
raise ValueError(f"Unsupported FP8 format: E{exp_bits}M{mantissa_bits}")
|
| 280 |
+
|
| 281 |
+
# Calculate FP8 max value
|
| 282 |
+
max_value = calculate_fp8_maxval(exp_bits, mantissa_bits)
|
| 283 |
+
min_value = -max_value # this function supports only signed FP8
|
| 284 |
+
|
| 285 |
+
# Define function to determine if a key is a target key. target means fp8 optimization, not for weight hook.
|
| 286 |
+
def is_target_key(key):
|
| 287 |
+
# Check if weight key matches target patterns and does not match exclude patterns
|
| 288 |
+
is_target = (target_layer_keys is None or any(pattern in key for pattern in target_layer_keys)) and key.endswith(".weight")
|
| 289 |
+
is_excluded = exclude_layer_keys is not None and any(pattern in key for pattern in exclude_layer_keys)
|
| 290 |
+
return is_target and not is_excluded
|
| 291 |
+
|
| 292 |
+
# Create optimized state dict
|
| 293 |
+
optimized_count = 0
|
| 294 |
+
|
| 295 |
+
# Process each file
|
| 296 |
+
state_dict = {}
|
| 297 |
+
for model_file in model_files:
|
| 298 |
+
with MemoryEfficientSafeOpen(model_file, disable_numpy_memmap=disable_numpy_memmap) as original_f:
|
| 299 |
+
f = TensorWeightAdapter(weight_transform_hooks, original_f) if weight_transform_hooks is not None else original_f
|
| 300 |
+
|
| 301 |
+
keys = f.keys()
|
| 302 |
+
for key in tqdm(keys, desc=f"Loading {os.path.basename(model_file)}", unit="key"):
|
| 303 |
+
value = f.get_tensor(key)
|
| 304 |
+
|
| 305 |
+
# Save original device
|
| 306 |
+
original_device = value.device # usually cpu
|
| 307 |
+
|
| 308 |
+
if weight_hook is not None:
|
| 309 |
+
# Apply weight hook if provided
|
| 310 |
+
value = weight_hook(key, value, keep_on_calc_device=(calc_device is not None))
|
| 311 |
+
|
| 312 |
+
if not is_target_key(key):
|
| 313 |
+
target_device = calc_device if (calc_device is not None and move_to_device) else original_device
|
| 314 |
+
value = value.to(target_device)
|
| 315 |
+
state_dict[key] = value
|
| 316 |
+
continue
|
| 317 |
+
|
| 318 |
+
# Move to calculation device
|
| 319 |
+
if calc_device is not None:
|
| 320 |
+
value = value.to(calc_device)
|
| 321 |
+
|
| 322 |
+
original_dtype = value.dtype
|
| 323 |
+
if original_dtype.itemsize == 1:
|
| 324 |
+
if allow_prequantized_fp8 and original_dtype in (torch.float8_e4m3fn, torch.float8_e5m2):
|
| 325 |
+
# Weight is already FP8-quantized (e.g. a pre-quantized ComfyUI checkpoint).
|
| 326 |
+
# Keep it as-is without re-quantizing; the matching scale is supplied separately
|
| 327 |
+
# (e.g. renamed to `.scale_weight` via weight_transform_hooks).
|
| 328 |
+
if not move_to_device:
|
| 329 |
+
value = value.to(original_device)
|
| 330 |
+
state_dict[key] = value
|
| 331 |
+
optimized_count += 1
|
| 332 |
+
continue
|
| 333 |
+
raise ValueError(
|
| 334 |
+
f"Layer {key} is already in {original_dtype} format. `--fp8_scaled` optimization should not be applied. Please use fp16/bf16/float32 model weights."
|
| 335 |
+
+ f" / レイヤー {key} は既に{original_dtype}形式です。`--fp8_scaled` 最適化は適用できません。FP16/BF16/Float32のモデル重みを使用してください。"
|
| 336 |
+
)
|
| 337 |
+
quantized_weight, scale_tensor = quantize_weight(
|
| 338 |
+
key, value, fp8_dtype, max_value, min_value, quantization_mode, block_size
|
| 339 |
+
)
|
| 340 |
+
|
| 341 |
+
# Add to state dict using original key for weight and new key for scale
|
| 342 |
+
fp8_key = key # Maintain original key
|
| 343 |
+
scale_key = key.replace(".weight", ".scale_weight")
|
| 344 |
+
assert fp8_key != scale_key, "FP8 key and scale key must be different"
|
| 345 |
+
|
| 346 |
+
if not move_to_device:
|
| 347 |
+
quantized_weight = quantized_weight.to(original_device)
|
| 348 |
+
|
| 349 |
+
# keep scale shape: [1] or [out,1] or [out, num_blocks, 1]. We can determine the quantization mode from the shape of scale_weight in the patched model.
|
| 350 |
+
scale_tensor = scale_tensor.to(dtype=original_dtype, device=quantized_weight.device)
|
| 351 |
+
|
| 352 |
+
state_dict[fp8_key] = quantized_weight
|
| 353 |
+
state_dict[scale_key] = scale_tensor
|
| 354 |
+
|
| 355 |
+
optimized_count += 1
|
| 356 |
+
|
| 357 |
+
if calc_device is not None and optimized_count % 10 == 0:
|
| 358 |
+
# free memory on calculation device
|
| 359 |
+
clean_memory_on_device(calc_device)
|
| 360 |
+
|
| 361 |
+
logger.info(f"Number of optimized Linear layers: {optimized_count}")
|
| 362 |
+
return state_dict
|
| 363 |
+
|
| 364 |
+
|
| 365 |
+
def fp8_linear_forward_patch(self: nn.Linear, x, use_scaled_mm=False, max_value=None):
|
| 366 |
+
"""
|
| 367 |
+
Patched forward method for Linear layers with FP8 weights.
|
| 368 |
+
|
| 369 |
+
Args:
|
| 370 |
+
self: Linear layer instance
|
| 371 |
+
x (torch.Tensor): Input tensor
|
| 372 |
+
use_scaled_mm (bool): Use scaled_mm for FP8 Linear layers, requires SM 8.9+ (RTX 40 series)
|
| 373 |
+
max_value (float): Maximum value for FP8 quantization. If None, no quantization is applied for input tensor.
|
| 374 |
+
|
| 375 |
+
Returns:
|
| 376 |
+
torch.Tensor: Result of linear transformation
|
| 377 |
+
"""
|
| 378 |
+
if use_scaled_mm:
|
| 379 |
+
# **not tested**
|
| 380 |
+
# _scaled_mm only works for per-tensor scale for now (per-channel scale does not work in certain cases)
|
| 381 |
+
if self.scale_weight.ndim != 1:
|
| 382 |
+
raise ValueError("scaled_mm only supports per-tensor scale_weight for now.")
|
| 383 |
+
|
| 384 |
+
input_dtype = x.dtype
|
| 385 |
+
original_weight_dtype = self.scale_weight.dtype
|
| 386 |
+
target_dtype = self.weight.dtype
|
| 387 |
+
# assert x.ndim == 3, "Input tensor must be 3D (batch_size, seq_len, hidden_dim)"
|
| 388 |
+
|
| 389 |
+
if max_value is None:
|
| 390 |
+
# no input quantization
|
| 391 |
+
scale_x = torch.tensor(1.0, dtype=torch.float32, device=x.device)
|
| 392 |
+
else:
|
| 393 |
+
# calculate scale factor for input tensor
|
| 394 |
+
scale_x = (torch.max(torch.abs(x.flatten())) / max_value).to(torch.float32)
|
| 395 |
+
|
| 396 |
+
# quantize input tensor to FP8: this seems to consume a lot of memory
|
| 397 |
+
fp8_max_value = torch.finfo(target_dtype).max
|
| 398 |
+
fp8_min_value = torch.finfo(target_dtype).min
|
| 399 |
+
x = quantize_fp8(x, scale_x, target_dtype, fp8_max_value, fp8_min_value)
|
| 400 |
+
|
| 401 |
+
original_shape = x.shape
|
| 402 |
+
x = x.reshape(-1, x.shape[-1]).to(target_dtype)
|
| 403 |
+
|
| 404 |
+
weight = self.weight.t()
|
| 405 |
+
scale_weight = self.scale_weight.to(torch.float32)
|
| 406 |
+
|
| 407 |
+
if self.bias is not None:
|
| 408 |
+
# float32 is not supported with bias in scaled_mm
|
| 409 |
+
o = torch._scaled_mm(x, weight, out_dtype=original_weight_dtype, bias=self.bias, scale_a=scale_x, scale_b=scale_weight)
|
| 410 |
+
else:
|
| 411 |
+
o = torch._scaled_mm(x, weight, out_dtype=input_dtype, scale_a=scale_x, scale_b=scale_weight)
|
| 412 |
+
|
| 413 |
+
o = o.reshape(original_shape[0], original_shape[1], -1) if len(original_shape) == 3 else o.reshape(original_shape[0], -1)
|
| 414 |
+
return o.to(input_dtype)
|
| 415 |
+
|
| 416 |
+
else:
|
| 417 |
+
# Dequantize the weight
|
| 418 |
+
original_dtype = self.scale_weight.dtype
|
| 419 |
+
if self.scale_weight.ndim < 3:
|
| 420 |
+
# per-tensor or per-channel quantization, we can broadcast
|
| 421 |
+
dequantized_weight = self.weight.to(original_dtype) * self.scale_weight
|
| 422 |
+
else:
|
| 423 |
+
# block-wise quantization, need to reshape weight to match scale shape for broadcasting
|
| 424 |
+
out_features, num_blocks, _ = self.scale_weight.shape
|
| 425 |
+
dequantized_weight = self.weight.to(original_dtype).contiguous().view(out_features, num_blocks, -1)
|
| 426 |
+
dequantized_weight = dequantized_weight * self.scale_weight
|
| 427 |
+
dequantized_weight = dequantized_weight.view(self.weight.shape)
|
| 428 |
+
|
| 429 |
+
# Perform linear transformation
|
| 430 |
+
dequantized_weight = dequantized_weight.to(x.dtype)
|
| 431 |
+
if self.bias is not None:
|
| 432 |
+
output = F.linear(x, dequantized_weight, self.bias)
|
| 433 |
+
else:
|
| 434 |
+
output = F.linear(x, dequantized_weight)
|
| 435 |
+
|
| 436 |
+
return output
|
| 437 |
+
|
| 438 |
+
|
| 439 |
+
def apply_fp8_monkey_patch(model, optimized_state_dict, use_scaled_mm=False):
|
| 440 |
+
"""
|
| 441 |
+
Apply monkey patching to a model using FP8 optimized state dict.
|
| 442 |
+
|
| 443 |
+
Args:
|
| 444 |
+
model (nn.Module): Model instance to patch
|
| 445 |
+
optimized_state_dict (dict): FP8 optimized state dict
|
| 446 |
+
use_scaled_mm (bool): Use scaled_mm for FP8 Linear layers, requires SM 8.9+ (RTX 40 series)
|
| 447 |
+
|
| 448 |
+
Returns:
|
| 449 |
+
nn.Module: The patched model (same instance, modified in-place)
|
| 450 |
+
"""
|
| 451 |
+
# # Calculate FP8 float8_e5m2 max value
|
| 452 |
+
# max_value = calculate_fp8_maxval(5, 2)
|
| 453 |
+
max_value = None # do not quantize input tensor
|
| 454 |
+
|
| 455 |
+
# Find all scale keys to identify FP8-optimized layers
|
| 456 |
+
scale_keys = [k for k in optimized_state_dict.keys() if k.endswith(".scale_weight")]
|
| 457 |
+
|
| 458 |
+
# Enumerate patched layers
|
| 459 |
+
patched_module_paths = set()
|
| 460 |
+
scale_shape_info = {}
|
| 461 |
+
for scale_key in scale_keys:
|
| 462 |
+
# Extract module path from scale key (remove .scale_weight)
|
| 463 |
+
module_path = scale_key.rsplit(".scale_weight", 1)[0]
|
| 464 |
+
patched_module_paths.add(module_path)
|
| 465 |
+
|
| 466 |
+
# Store scale shape information
|
| 467 |
+
scale_shape_info[module_path] = optimized_state_dict[scale_key].shape
|
| 468 |
+
|
| 469 |
+
patched_count = 0
|
| 470 |
+
|
| 471 |
+
# Apply monkey patch to each layer with FP8 weights
|
| 472 |
+
for name, module in model.named_modules():
|
| 473 |
+
# Check if this module has a corresponding scale_weight
|
| 474 |
+
has_scale = name in patched_module_paths
|
| 475 |
+
|
| 476 |
+
# Apply patch if it's a Linear layer with FP8 scale
|
| 477 |
+
if isinstance(module, nn.Linear) and has_scale:
|
| 478 |
+
# register the scale_weight as a buffer to load the state_dict
|
| 479 |
+
# module.register_buffer("scale_weight", torch.tensor(1.0, dtype=module.weight.dtype))
|
| 480 |
+
scale_shape = scale_shape_info[name]
|
| 481 |
+
module.register_buffer("scale_weight", torch.ones(scale_shape, dtype=module.weight.dtype))
|
| 482 |
+
|
| 483 |
+
# Create a new forward method with the patched version.
|
| 484 |
+
def new_forward(self, x):
|
| 485 |
+
return fp8_linear_forward_patch(self, x, use_scaled_mm, max_value)
|
| 486 |
+
|
| 487 |
+
# Bind method to module
|
| 488 |
+
module.forward = new_forward.__get__(module, type(module))
|
| 489 |
+
|
| 490 |
+
patched_count += 1
|
| 491 |
+
|
| 492 |
+
logger.info(f"Number of monkey-patched Linear layers: {patched_count}")
|
| 493 |
+
return model
|
| 494 |
+
|
| 495 |
+
|
| 496 |
+
# Example usage
|
| 497 |
+
def example_usage():
|
| 498 |
+
# Small test model
|
| 499 |
+
class TestModel(nn.Module):
|
| 500 |
+
def __init__(self):
|
| 501 |
+
super().__init__()
|
| 502 |
+
fc1 = nn.Linear(768, 3072)
|
| 503 |
+
act1 = nn.GELU()
|
| 504 |
+
fc2 = nn.Linear(3072, 768)
|
| 505 |
+
act2 = nn.GELU()
|
| 506 |
+
fc3 = nn.Linear(768, 768)
|
| 507 |
+
|
| 508 |
+
# Set layer names for testing
|
| 509 |
+
self.single_blocks = nn.ModuleList([fc1, act1, fc2, act2, fc3])
|
| 510 |
+
|
| 511 |
+
self.fc4 = nn.Linear(768, 128)
|
| 512 |
+
|
| 513 |
+
def forward(self, x):
|
| 514 |
+
for layer in self.single_blocks:
|
| 515 |
+
x = layer(x)
|
| 516 |
+
x = self.fc4(x)
|
| 517 |
+
return x
|
| 518 |
+
|
| 519 |
+
# Instantiate model
|
| 520 |
+
test_model = TestModel()
|
| 521 |
+
test_model.to(torch.float16) # convert to FP16 for testing
|
| 522 |
+
|
| 523 |
+
# Test input tensor
|
| 524 |
+
test_input = torch.randn(1, 768, dtype=torch.float16)
|
| 525 |
+
|
| 526 |
+
# Calculate output before optimization
|
| 527 |
+
with torch.no_grad():
|
| 528 |
+
original_output = test_model(test_input)
|
| 529 |
+
print("original output", original_output[0, :5])
|
| 530 |
+
|
| 531 |
+
# Get state dict
|
| 532 |
+
state_dict = test_model.state_dict()
|
| 533 |
+
|
| 534 |
+
# Apply FP8 optimization to state dict
|
| 535 |
+
cuda_device = torch.device("cuda")
|
| 536 |
+
optimized_state_dict = optimize_state_dict_with_fp8(state_dict, cuda_device, ["single_blocks"], ["2"])
|
| 537 |
+
|
| 538 |
+
# Apply monkey patching to the model
|
| 539 |
+
optimized_model = TestModel() # re-instantiate model
|
| 540 |
+
optimized_model.to(torch.float16) # convert to FP16 for testing
|
| 541 |
+
apply_fp8_monkey_patch(optimized_model, optimized_state_dict)
|
| 542 |
+
|
| 543 |
+
# Load optimized state dict
|
| 544 |
+
optimized_model.load_state_dict(optimized_state_dict, strict=True, assign=True) # assign=True to load buffer
|
| 545 |
+
|
| 546 |
+
# Calculate output after optimization
|
| 547 |
+
with torch.no_grad():
|
| 548 |
+
optimized_output = optimized_model(test_input)
|
| 549 |
+
print("optimized output", optimized_output[0, :5])
|
| 550 |
+
|
| 551 |
+
# Compare accuracy
|
| 552 |
+
error = torch.mean(torch.abs(original_output - optimized_output))
|
| 553 |
+
print(f"Mean absolute error: {error.item()}")
|
| 554 |
+
|
| 555 |
+
# Check memory usage
|
| 556 |
+
original_params = sum(p.nelement() * p.element_size() for p in test_model.parameters()) / (1024 * 1024)
|
| 557 |
+
print(f"Model parameter memory: {original_params:.2f} MB")
|
| 558 |
+
optimized_params = sum(p.nelement() * p.element_size() for p in optimized_model.parameters()) / (1024 * 1024)
|
| 559 |
+
print(f"Optimized model parameter memory: {optimized_params:.2f} MB")
|
| 560 |
+
|
| 561 |
+
return test_model
|
| 562 |
+
|
| 563 |
+
|
| 564 |
+
if __name__ == "__main__":
|
| 565 |
+
example_usage()
|
third_party/musubi_tuner/modules/nvfp4_utils.py
ADDED
|
@@ -0,0 +1,518 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Low-level NVFP4 routines (E2M1 encode/decode, cuBLAS swizzle, scaled_mm call) are
|
| 2 |
+
# adapted from the exp-nvfp4-support-for-torch-2-10 branch, which was inspired by
|
| 3 |
+
# comfy-kitchen (https://github.com/Comfy-Org/comfy-kitchen, Apache License 2.0).
|
| 4 |
+
|
| 5 |
+
"""Loading ComfyUI pre-quantized NVFP4 (+AWQ) checkpoints for frozen text encoders.
|
| 6 |
+
|
| 7 |
+
Layout of a ComfyUI NVFP4 artifact (verified bit-level against the BF16 reference of
|
| 8 |
+
the MiniMax-H3 Qwen3-VL text encoder):
|
| 9 |
+
|
| 10 |
+
- NVFP4 Linear: ``.weight`` U8 [N, K/2] with two E2M1 codes per byte (element 0 in the
|
| 11 |
+
HIGH nibble), ``.weight_scale`` F8_E4M3 per-16-block scales stored in the cuBLAS
|
| 12 |
+
128x4 tiled ("swizzled") layout, ``.weight_scale_2`` F32 per-tensor scale, and a
|
| 13 |
+
``.comfy_quant`` spec ``{"format": "nvfp4", ...}``. Modules quantized with AWQ carry
|
| 14 |
+
a ``.pre_quant_scale`` [K] tensor that is multiplied into the *input* at runtime;
|
| 15 |
+
for the remaining modules the AWQ scale is folded into the preceding norm weights,
|
| 16 |
+
so the checkpoint is self-consistent and needs no special handling here.
|
| 17 |
+
- INT8 embedding: ``.weight`` I8 [V, D] + ``.weight_scale`` F32 [V, 1]
|
| 18 |
+
(``{"format": "int8_tensorwise"}``, effectively per-row); dequant = weight * scale.
|
| 19 |
+
- Everything else (norms, vision tower, biases) stays BF16.
|
| 20 |
+
|
| 21 |
+
The state dict is converted to the Musubi runtime layout: ``.weight_scale`` becomes
|
| 22 |
+
``.nvfp4_block_scale`` (kept swizzled: ``torch.nn.functional.scaled_mm`` consumes the
|
| 23 |
+
swizzled layout directly, the dequantizing fallback unswizzles on the fly) and
|
| 24 |
+
``.weight_scale_2`` becomes ``.nvfp4_scale``; the embedding scale becomes
|
| 25 |
+
``.scale_weight`` (same dequant semantics as the ConvRot INT8 layout).
|
| 26 |
+
|
| 27 |
+
Inference only: NVFP4 modules are frozen, the patched forwards have no autograd
|
| 28 |
+
support. Dynamic (on-the-fly) NVFP4 quantization is deliberately not offered — the
|
| 29 |
+
published artifacts are AWQ-calibrated, which cannot be reproduced without
|
| 30 |
+
calibration data, and dynamically quantizing BF16 weights would silently produce a
|
| 31 |
+
lower-quality model than ConvRot INT8.
|
| 32 |
+
"""
|
| 33 |
+
|
| 34 |
+
import os
|
| 35 |
+
from typing import Dict, List, Optional, Tuple, Union
|
| 36 |
+
|
| 37 |
+
import torch
|
| 38 |
+
import torch.nn as nn
|
| 39 |
+
import torch.nn.functional as F
|
| 40 |
+
|
| 41 |
+
import logging
|
| 42 |
+
|
| 43 |
+
from tqdm import tqdm
|
| 44 |
+
|
| 45 |
+
from musubi_tuner.modules.comfy_quant_utils import (
|
| 46 |
+
COMFY_QUANT_SUFFIX,
|
| 47 |
+
COMFY_WEIGHT_SCALE_SUFFIX,
|
| 48 |
+
FORMAT_INT8_TENSORWISE,
|
| 49 |
+
FORMAT_NVFP4,
|
| 50 |
+
classify_comfy_quant_spec,
|
| 51 |
+
decode_comfy_quant_spec,
|
| 52 |
+
)
|
| 53 |
+
from musubi_tuner.utils.safetensors_utils import MemoryEfficientSafeOpen, TensorWeightAdapter, WeightTransformHooks
|
| 54 |
+
|
| 55 |
+
logger = logging.getLogger(__name__)
|
| 56 |
+
|
| 57 |
+
NVFP4_BLOCK_SIZE = 16
|
| 58 |
+
|
| 59 |
+
F4_E2M1_MAX = 6.0
|
| 60 |
+
F8_E4M3_MAX = 448.0
|
| 61 |
+
|
| 62 |
+
COMFY_WEIGHT_SCALE_2_SUFFIX = ".weight_scale_2"
|
| 63 |
+
COMFY_PRE_QUANT_SCALE_SUFFIX = ".pre_quant_scale"
|
| 64 |
+
|
| 65 |
+
# E2M1 code -> value (codes 0..7 positive, 8..15 negative)
|
| 66 |
+
_E2M1_VALUES = (0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0, -0.0, -0.5, -1.0, -1.5, -2.0, -3.0, -4.0, -6.0)
|
| 67 |
+
|
| 68 |
+
# byte -> (high nibble value, low nibble value), cached per (device, dtype)
|
| 69 |
+
_BYTE_PAIR_LUT_CACHE: Dict[Tuple[torch.device, torch.dtype], torch.Tensor] = {}
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def _byte_pair_lut(device: torch.device, dtype: torch.dtype) -> torch.Tensor:
|
| 73 |
+
lut = _BYTE_PAIR_LUT_CACHE.get((device, dtype))
|
| 74 |
+
if lut is None:
|
| 75 |
+
values = torch.tensor(_E2M1_VALUES, dtype=torch.float32)
|
| 76 |
+
codes = torch.arange(256, dtype=torch.int64)
|
| 77 |
+
lut = torch.stack([values[codes >> 4], values[codes & 0x0F]], dim=1).to(device=device, dtype=dtype)
|
| 78 |
+
_BYTE_PAIR_LUT_CACHE[(device, dtype)] = lut
|
| 79 |
+
return lut
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
def _ceil_div(a: int, b: int) -> int:
|
| 83 |
+
return (a + b - 1) // b
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
def _roundup(value: int, multiple: int) -> int:
|
| 87 |
+
return _ceil_div(value, multiple) * multiple
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
# region cuBLAS 128x4 tiled block-scale layout
|
| 91 |
+
# https://docs.nvidia.com/cuda/cublas/index.html#d-block-scaling-factors-layout
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
def to_blocked(input_matrix: torch.Tensor) -> torch.Tensor:
|
| 95 |
+
"""Rearrange (H, W) block scales into the cuBLAS tiled layout (padded to 128x4)."""
|
| 96 |
+
rows, cols = input_matrix.shape
|
| 97 |
+
n_row_blocks = _ceil_div(rows, 128)
|
| 98 |
+
n_col_blocks = _ceil_div(cols, 4)
|
| 99 |
+
padded_rows = n_row_blocks * 128
|
| 100 |
+
padded_cols = n_col_blocks * 4
|
| 101 |
+
padded = input_matrix
|
| 102 |
+
if (rows, cols) != (padded_rows, padded_cols):
|
| 103 |
+
padded = torch.zeros((padded_rows, padded_cols), device=input_matrix.device, dtype=input_matrix.dtype)
|
| 104 |
+
padded[:rows, :cols] = input_matrix
|
| 105 |
+
blocks = padded.view(n_row_blocks, 128, n_col_blocks, 4).permute(0, 2, 1, 3)
|
| 106 |
+
rearranged = blocks.reshape(-1, 4, 32, 4).transpose(1, 2).reshape(-1, 32, 16)
|
| 107 |
+
return rearranged.reshape(padded_rows, padded_cols)
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
def from_blocked(blocked_matrix: torch.Tensor, num_rows: int, num_cols: int) -> torch.Tensor:
|
| 111 |
+
"""Reverse the cuBLAS tiled layout back to a row-major (num_rows, num_cols) matrix."""
|
| 112 |
+
n_row_blocks = _ceil_div(num_rows, 128)
|
| 113 |
+
n_col_blocks = _ceil_div(num_cols, 4)
|
| 114 |
+
padded_rows = n_row_blocks * 128
|
| 115 |
+
padded_cols = n_col_blocks * 4
|
| 116 |
+
step1 = blocked_matrix.reshape(-1, 32, 16)
|
| 117 |
+
step2 = step1.reshape(-1, 32, 4, 4).transpose(1, 2)
|
| 118 |
+
step3 = step2.reshape(n_row_blocks, n_col_blocks, 4, 32, 4)
|
| 119 |
+
step4 = step3.reshape(n_row_blocks, n_col_blocks, 128, 4)
|
| 120 |
+
step5 = step4.permute(0, 2, 1, 3)
|
| 121 |
+
unblocked = step5.reshape(padded_rows, padded_cols)
|
| 122 |
+
return unblocked[:num_rows, :num_cols]
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
# endregion
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
def dequantize_nvfp4(
|
| 129 |
+
packed: torch.Tensor,
|
| 130 |
+
block_scale: torch.Tensor,
|
| 131 |
+
per_tensor_scale: torch.Tensor,
|
| 132 |
+
orig_shape: Tuple[int, int],
|
| 133 |
+
out_dtype: torch.dtype,
|
| 134 |
+
) -> torch.Tensor:
|
| 135 |
+
"""Dequantize packed NVFP4 data (block_scale in the swizzled layout) to ``orig_shape``."""
|
| 136 |
+
stored_rows = packed.shape[0]
|
| 137 |
+
stored_cols = packed.shape[1] * 2
|
| 138 |
+
lut = _byte_pair_lut(packed.device, out_dtype)
|
| 139 |
+
# one embedding lookup decodes both nibbles of each byte: [R, C/2] -> [R, C/2, 2] -> [R, C]
|
| 140 |
+
weight = F.embedding(packed.reshape(-1).int(), lut).reshape(stored_rows, stored_cols)
|
| 141 |
+
scales = from_blocked(block_scale, stored_rows, stored_cols // NVFP4_BLOCK_SIZE)
|
| 142 |
+
total = (per_tensor_scale.float() * scales.float()).to(out_dtype)
|
| 143 |
+
weight = (weight.reshape(stored_rows, -1, NVFP4_BLOCK_SIZE) * total.unsqueeze(-1)).reshape(stored_rows, stored_cols)
|
| 144 |
+
return weight[: orig_shape[0], : orig_shape[1]]
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
# region runtime activation quantization for the scaled_mm (W4A4) path
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
def _n_ones(n: int) -> int:
|
| 151 |
+
return (1 << n) - 1
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
_EBITS_F32, _MBITS_F32 = 8, 23
|
| 155 |
+
_F32_EXP_BIAS = _n_ones(_EBITS_F32 - 1)
|
| 156 |
+
|
| 157 |
+
|
| 158 |
+
def _f32_to_e2m1_unpacked(x: torch.Tensor) -> torch.Tensor:
|
| 159 |
+
"""Convert FP32 to E2M1 codes stored one-per-byte in uint8 (round-to-nearest-even)."""
|
| 160 |
+
ebits, mbits = 2, 1
|
| 161 |
+
assert x.dtype == torch.float
|
| 162 |
+
exp_bias = _n_ones(ebits - 1)
|
| 163 |
+
max_int = _n_ones(ebits + mbits)
|
| 164 |
+
sign_mask = 1 << (ebits + mbits)
|
| 165 |
+
magic_adder = _n_ones(_MBITS_F32 - mbits - 1)
|
| 166 |
+
max_normal = 2 ** (_n_ones(ebits) - exp_bias) * (_n_ones(mbits + 1) / (2**mbits))
|
| 167 |
+
min_normal = 2 ** (1 - exp_bias)
|
| 168 |
+
denorm_exp = (_F32_EXP_BIAS - exp_bias) + (_MBITS_F32 - mbits) + 1
|
| 169 |
+
denorm_mask_int = denorm_exp << _MBITS_F32
|
| 170 |
+
denorm_mask_float = torch.tensor(denorm_mask_int, dtype=torch.int32, device=x.device).view(torch.float32)
|
| 171 |
+
|
| 172 |
+
x = x.view(torch.int32)
|
| 173 |
+
sign = x & 0x80000000
|
| 174 |
+
x = (x ^ sign).view(torch.float)
|
| 175 |
+
|
| 176 |
+
saturate_mask = x >= max_normal
|
| 177 |
+
denormal_mask = torch.logical_and(torch.logical_not(saturate_mask), x < min_normal)
|
| 178 |
+
normal_mask = torch.logical_not(torch.logical_or(saturate_mask, denormal_mask))
|
| 179 |
+
|
| 180 |
+
denormal_x = (x + denorm_mask_float).view(torch.int32) - denorm_mask_int
|
| 181 |
+
denormal_x = denormal_x.to(torch.uint8)
|
| 182 |
+
|
| 183 |
+
normal_x = x.view(torch.int32)
|
| 184 |
+
mant_odd = (normal_x >> (_MBITS_F32 - mbits)) & 1
|
| 185 |
+
normal_x = normal_x + (((exp_bias - _F32_EXP_BIAS) << _MBITS_F32) + magic_adder) + mant_odd
|
| 186 |
+
normal_x = (normal_x >> (_MBITS_F32 - mbits)).to(torch.uint8)
|
| 187 |
+
|
| 188 |
+
result = torch.full_like(x, max_int, dtype=torch.uint8)
|
| 189 |
+
result = torch.where(denormal_mask, denormal_x, result)
|
| 190 |
+
result = torch.where(normal_mask, normal_x, result)
|
| 191 |
+
|
| 192 |
+
sign_lp = (sign >> (_MBITS_F32 + _EBITS_F32 - mbits - ebits)).to(torch.uint8) & sign_mask
|
| 193 |
+
return result | sign_lp
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
def pack_uint4(codes: torch.Tensor) -> torch.Tensor:
|
| 197 |
+
"""Pack pairs of 4-bit codes (one per byte) into bytes, element 0 in the HIGH nibble."""
|
| 198 |
+
shape = codes.shape
|
| 199 |
+
assert shape[-1] % 2 == 0
|
| 200 |
+
codes = codes.contiguous().view(-1)
|
| 201 |
+
return (codes[::2] << 4 | codes[1::2]).view(*shape[:-1], shape[-1] // 2)
|
| 202 |
+
|
| 203 |
+
|
| 204 |
+
def quantize_nvfp4_activation(x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, int]:
|
| 205 |
+
"""Quantize a 2D activation to NVFP4 for scaled_mm.
|
| 206 |
+
|
| 207 |
+
Returns (packed uint8 [Mp, K/2], swizzled block scales F8_E4M3, per-tensor scale F32,
|
| 208 |
+
original row count). Rows are padded to a multiple of 16 as scaled_mm requires.
|
| 209 |
+
"""
|
| 210 |
+
orig_rows, cols = x.shape
|
| 211 |
+
if cols % NVFP4_BLOCK_SIZE != 0:
|
| 212 |
+
raise ValueError(f"NVFP4 activation width must be a multiple of {NVFP4_BLOCK_SIZE}, got {cols}")
|
| 213 |
+
padded_rows = _roundup(orig_rows, 16)
|
| 214 |
+
if padded_rows != orig_rows:
|
| 215 |
+
x = F.pad(x, (0, 0, 0, padded_rows - orig_rows))
|
| 216 |
+
|
| 217 |
+
per_tensor_scale = (torch.amax(x.abs()).float() / (F8_E4M3_MAX * F4_E2M1_MAX)).reshape(())
|
| 218 |
+
|
| 219 |
+
blocks = x.reshape(padded_rows, -1, NVFP4_BLOCK_SIZE)
|
| 220 |
+
block_scale = torch.amax(blocks.abs(), dim=-1).float() / F4_E2M1_MAX
|
| 221 |
+
scaled = torch.clamp(block_scale / torch.clamp(per_tensor_scale, min=torch.finfo(torch.float32).tiny), max=F8_E4M3_MAX)
|
| 222 |
+
scaled_f8 = scaled.to(torch.float8_e4m3fn)
|
| 223 |
+
total = per_tensor_scale * scaled_f8.float()
|
| 224 |
+
total_safe = torch.where(total == 0, torch.ones_like(total), total)
|
| 225 |
+
|
| 226 |
+
data = blocks.float() / total_safe.unsqueeze(-1)
|
| 227 |
+
data = torch.where((total == 0).unsqueeze(-1), torch.zeros_like(data), data)
|
| 228 |
+
data = torch.clamp(data, -F4_E2M1_MAX, F4_E2M1_MAX).reshape(padded_rows, cols)
|
| 229 |
+
|
| 230 |
+
packed = pack_uint4(_f32_to_e2m1_unpacked(data))
|
| 231 |
+
return packed, to_blocked(scaled_f8), per_tensor_scale, orig_rows
|
| 232 |
+
|
| 233 |
+
|
| 234 |
+
def nvfp4_scaled_mm_available() -> bool:
|
| 235 |
+
return hasattr(torch, "float4_e2m1fn_x2") and hasattr(torch.nn.functional, "scaled_mm")
|
| 236 |
+
|
| 237 |
+
|
| 238 |
+
def nvfp4_scaled_mm_linear(
|
| 239 |
+
x: torch.Tensor,
|
| 240 |
+
weight_packed: torch.Tensor,
|
| 241 |
+
weight_block_scale: torch.Tensor,
|
| 242 |
+
weight_scale: torch.Tensor,
|
| 243 |
+
bias: Optional[torch.Tensor],
|
| 244 |
+
orig_out_features: int,
|
| 245 |
+
) -> torch.Tensor:
|
| 246 |
+
"""W4A4 linear via torch.nn.functional.scaled_mm (torch 2.10+, Blackwell)."""
|
| 247 |
+
from torch.nn.functional import ScalingType, SwizzleType
|
| 248 |
+
|
| 249 |
+
x_packed, x_block_scale, x_scale, orig_rows = quantize_nvfp4_activation(x)
|
| 250 |
+
result = torch.nn.functional.scaled_mm(
|
| 251 |
+
x_packed.view(torch.float4_e2m1fn_x2),
|
| 252 |
+
weight_packed.view(torch.float4_e2m1fn_x2).t(),
|
| 253 |
+
scale_a=[x_block_scale.view(-1), x_scale],
|
| 254 |
+
scale_b=[weight_block_scale.view(-1), weight_scale],
|
| 255 |
+
bias=bias,
|
| 256 |
+
output_dtype=x.dtype,
|
| 257 |
+
scale_recipe_a=[ScalingType.BlockWise1x16, ScalingType.TensorWise],
|
| 258 |
+
scale_recipe_b=[ScalingType.BlockWise1x16, ScalingType.TensorWise],
|
| 259 |
+
swizzle_a=[SwizzleType.SWIZZLE_32_4_4, SwizzleType.NO_SWIZZLE],
|
| 260 |
+
swizzle_b=[SwizzleType.SWIZZLE_32_4_4, SwizzleType.NO_SWIZZLE],
|
| 261 |
+
)
|
| 262 |
+
return result[:orig_rows, :orig_out_features]
|
| 263 |
+
|
| 264 |
+
|
| 265 |
+
# endregion
|
| 266 |
+
|
| 267 |
+
|
| 268 |
+
# region pre-quantized state dict loading
|
| 269 |
+
|
| 270 |
+
|
| 271 |
+
class NvFp4Quantizer:
|
| 272 |
+
"""Streams a ComfyUI pre-quantized NVFP4 (+ INT8 embedding) checkpoint.
|
| 273 |
+
|
| 274 |
+
Same protocol as ``ConvRotInt8Quantizer``: passed as ``quantizer`` to
|
| 275 |
+
``load_safetensors_with_lora_and_fp8``. Loading only converts key names and
|
| 276 |
+
validates the tensors — there is no dynamic quantization, the file dictates the
|
| 277 |
+
quantized layers. ``nvfp4_module_shapes`` maps module paths to their original
|
| 278 |
+
(out_features, in_features) after loading; ``int8_embedding_modules`` lists the
|
| 279 |
+
per-row INT8 modules (embeddings). Pass both to ``apply_nvfp4_monkey_patch``.
|
| 280 |
+
"""
|
| 281 |
+
|
| 282 |
+
def __init__(self):
|
| 283 |
+
self.nvfp4_module_shapes: Dict[str, Tuple[int, int]] = {}
|
| 284 |
+
self.int8_embedding_modules: List[str] = []
|
| 285 |
+
|
| 286 |
+
def load_and_quantize(
|
| 287 |
+
self,
|
| 288 |
+
model_files: List[str],
|
| 289 |
+
calc_device: Union[str, torch.device, None],
|
| 290 |
+
move_to_device: bool = False,
|
| 291 |
+
weight_hook: Optional[callable] = None,
|
| 292 |
+
disable_numpy_memmap: bool = False,
|
| 293 |
+
weight_transform_hooks: Optional[WeightTransformHooks] = None,
|
| 294 |
+
) -> dict:
|
| 295 |
+
state_dict = {}
|
| 296 |
+
module_formats: Dict[str, str] = {} # spans all shards
|
| 297 |
+
for model_file in model_files:
|
| 298 |
+
with MemoryEfficientSafeOpen(model_file, disable_numpy_memmap=disable_numpy_memmap) as original_f:
|
| 299 |
+
f = TensorWeightAdapter(weight_transform_hooks, original_f) if weight_transform_hooks is not None else original_f
|
| 300 |
+
|
| 301 |
+
keys = f.keys()
|
| 302 |
+
|
| 303 |
+
# pre-scan the tiny spec tensors so each module's format is known before
|
| 304 |
+
# the (possibly earlier-iterated) weight/scale keys arrive
|
| 305 |
+
for key in keys:
|
| 306 |
+
if key.endswith(COMFY_QUANT_SUFFIX):
|
| 307 |
+
module_path = key[: -len(COMFY_QUANT_SUFFIX)]
|
| 308 |
+
spec_format = classify_comfy_quant_spec(decode_comfy_quant_spec(key, f.get_tensor(key)))
|
| 309 |
+
if spec_format not in (FORMAT_NVFP4, FORMAT_INT8_TENSORWISE):
|
| 310 |
+
raise ValueError(
|
| 311 |
+
f"Unsupported comfy_quant format for {key}: {spec_format}. The NVFP4 loader supports"
|
| 312 |
+
' "nvfp4" Linear layers and "int8_tensorwise" embeddings only.'
|
| 313 |
+
f" / {key} の comfy_quant 形式 {spec_format} はNVFP4ローダーではサポートされていません。"
|
| 314 |
+
)
|
| 315 |
+
module_formats[module_path] = spec_format
|
| 316 |
+
if module_formats and weight_hook is not None:
|
| 317 |
+
raise ValueError(
|
| 318 |
+
f"Cannot merge LoRA weights into pre-quantized NVFP4 checkpoint {model_file}."
|
| 319 |
+
" Use the original BF16 weights instead."
|
| 320 |
+
f" / 事前量子化済みNVFP4チェックポイント {model_file} にはLoRAをマージできません。"
|
| 321 |
+
"BF16の元重みを使用してください。"
|
| 322 |
+
)
|
| 323 |
+
|
| 324 |
+
for key in tqdm(keys, desc=f"Loading {os.path.basename(model_file)}", unit="key"):
|
| 325 |
+
if key.endswith(COMFY_QUANT_SUFFIX):
|
| 326 |
+
continue # consumed in the pre-scan, not a model tensor
|
| 327 |
+
|
| 328 |
+
value = f.get_tensor(key)
|
| 329 |
+
original_device = value.device # usually cpu
|
| 330 |
+
passthrough_device = calc_device if (calc_device is not None and move_to_device) else original_device
|
| 331 |
+
converted_key = self._convert_key(key, value, module_formats)
|
| 332 |
+
state_dict[converted_key] = value.to(passthrough_device)
|
| 333 |
+
|
| 334 |
+
self._validate_completeness(state_dict, module_formats)
|
| 335 |
+
logger.info(
|
| 336 |
+
f"Number of pre-quantized layers: {len(self.nvfp4_module_shapes)} NVFP4 Linear,"
|
| 337 |
+
f" {len(self.int8_embedding_modules)} INT8 embedding"
|
| 338 |
+
)
|
| 339 |
+
return state_dict
|
| 340 |
+
|
| 341 |
+
def _convert_key(self, key: str, value: torch.Tensor, module_formats: Dict[str, str]) -> str:
|
| 342 |
+
"""Validate a tensor against its module's declared format and return the Musubi key."""
|
| 343 |
+
for suffix in (COMFY_WEIGHT_SCALE_SUFFIX, COMFY_WEIGHT_SCALE_2_SUFFIX, COMFY_PRE_QUANT_SCALE_SUFFIX, ".weight"):
|
| 344 |
+
if key.endswith(suffix):
|
| 345 |
+
module_path = key[: -len(suffix)]
|
| 346 |
+
break
|
| 347 |
+
else:
|
| 348 |
+
return key # bias, norm, etc.: passthrough
|
| 349 |
+
spec_format = module_formats.get(module_path)
|
| 350 |
+
if spec_format is None:
|
| 351 |
+
if key.endswith(".weight") and value.dtype.itemsize == 1:
|
| 352 |
+
raise ValueError(
|
| 353 |
+
f"Layer {key} is already in {value.dtype} format but has no {COMFY_QUANT_SUFFIX} spec."
|
| 354 |
+
f" / レイヤー {key} は既に{value.dtype}形式ですが {COMFY_QUANT_SUFFIX} がありません。"
|
| 355 |
+
)
|
| 356 |
+
if key.endswith((COMFY_WEIGHT_SCALE_SUFFIX, COMFY_WEIGHT_SCALE_2_SUFFIX, COMFY_PRE_QUANT_SCALE_SUFFIX)):
|
| 357 |
+
raise ValueError(f"Found {key} without a matching {module_path}{COMFY_QUANT_SUFFIX} spec")
|
| 358 |
+
return key
|
| 359 |
+
|
| 360 |
+
if spec_format == FORMAT_NVFP4:
|
| 361 |
+
if key.endswith(COMFY_WEIGHT_SCALE_SUFFIX):
|
| 362 |
+
if value.dtype is not torch.float8_e4m3fn:
|
| 363 |
+
raise ValueError(f"NVFP4 block scale {key} must be F8_E4M3, got {value.dtype}")
|
| 364 |
+
return module_path + ".nvfp4_block_scale"
|
| 365 |
+
if key.endswith(COMFY_WEIGHT_SCALE_2_SUFFIX):
|
| 366 |
+
if value.dtype is not torch.float32 or value.ndim != 0:
|
| 367 |
+
raise ValueError(f"NVFP4 per-tensor scale {key} must be a F32 scalar, got {value.dtype} {tuple(value.shape)}")
|
| 368 |
+
return module_path + ".nvfp4_scale"
|
| 369 |
+
if key.endswith(COMFY_PRE_QUANT_SCALE_SUFFIX):
|
| 370 |
+
if not value.is_floating_point() or value.ndim != 1:
|
| 371 |
+
raise ValueError(f"AWQ pre_quant_scale {key} must be a 1D float tensor, got {value.dtype} {tuple(value.shape)}")
|
| 372 |
+
return key
|
| 373 |
+
# .weight
|
| 374 |
+
if value.dtype is not torch.uint8 or value.ndim != 2:
|
| 375 |
+
raise ValueError(f"NVFP4 weight {key} must be 2D uint8 (packed), got {value.dtype} ndim={value.ndim}")
|
| 376 |
+
in_features = value.shape[1] * 2
|
| 377 |
+
if in_features % NVFP4_BLOCK_SIZE != 0:
|
| 378 |
+
raise ValueError(f"NVFP4 weight {key}: in_features {in_features} not a multiple of {NVFP4_BLOCK_SIZE}")
|
| 379 |
+
self.nvfp4_module_shapes[module_path] = (value.shape[0], in_features)
|
| 380 |
+
return key
|
| 381 |
+
|
| 382 |
+
# FORMAT_INT8_TENSORWISE: per-row INT8, embeddings only (validated at patch time)
|
| 383 |
+
if key.endswith(COMFY_WEIGHT_SCALE_SUFFIX):
|
| 384 |
+
if value.dtype is not torch.float32:
|
| 385 |
+
raise ValueError(f"INT8 per-row scale {key} must be F32, got {value.dtype}")
|
| 386 |
+
return module_path + ".scale_weight"
|
| 387 |
+
if key.endswith((COMFY_WEIGHT_SCALE_2_SUFFIX, COMFY_PRE_QUANT_SCALE_SUFFIX)):
|
| 388 |
+
raise ValueError(f"Unexpected tensor {key} for int8_tensorwise module {module_path}")
|
| 389 |
+
if value.dtype is not torch.int8:
|
| 390 |
+
raise ValueError(f"INT8 weight {key} must be int8, got {value.dtype}")
|
| 391 |
+
self.int8_embedding_modules.append(module_path)
|
| 392 |
+
return key
|
| 393 |
+
|
| 394 |
+
def _validate_completeness(self, state_dict: dict, module_formats: Dict[str, str]) -> None:
|
| 395 |
+
for module_path, spec_format in module_formats.items():
|
| 396 |
+
if spec_format == FORMAT_NVFP4:
|
| 397 |
+
required = (".weight", ".nvfp4_block_scale", ".nvfp4_scale")
|
| 398 |
+
else:
|
| 399 |
+
required = (".weight", ".scale_weight")
|
| 400 |
+
missing = [module_path + suffix for suffix in required if module_path + suffix not in state_dict]
|
| 401 |
+
if missing:
|
| 402 |
+
raise ValueError(f"Pre-quantized module {module_path} is missing tensors {missing}")
|
| 403 |
+
|
| 404 |
+
if spec_format == FORMAT_NVFP4:
|
| 405 |
+
rows, in_features = self.nvfp4_module_shapes[module_path]
|
| 406 |
+
expected_scale_numel = _roundup(rows, 128) * _roundup(in_features // NVFP4_BLOCK_SIZE, 4)
|
| 407 |
+
block_scale = state_dict[module_path + ".nvfp4_block_scale"]
|
| 408 |
+
if block_scale.numel() != expected_scale_numel:
|
| 409 |
+
raise ValueError(
|
| 410 |
+
f"NVFP4 module {module_path}: block scale has {block_scale.numel()} elements,"
|
| 411 |
+
f" expected {expected_scale_numel} for weight shape ({rows}, {in_features})"
|
| 412 |
+
)
|
| 413 |
+
pre_quant_scale = state_dict.get(module_path + COMFY_PRE_QUANT_SCALE_SUFFIX)
|
| 414 |
+
if pre_quant_scale is not None and pre_quant_scale.shape[0] != in_features:
|
| 415 |
+
raise ValueError(
|
| 416 |
+
f"NVFP4 module {module_path}: pre_quant_scale has {pre_quant_scale.shape[0]} elements,"
|
| 417 |
+
f" expected in_features {in_features}"
|
| 418 |
+
)
|
| 419 |
+
else:
|
| 420 |
+
weight = state_dict[module_path + ".weight"]
|
| 421 |
+
scale = state_dict[module_path + ".scale_weight"]
|
| 422 |
+
expected_scale_shape = (weight.shape[0], 1)
|
| 423 |
+
if tuple(scale.shape) != expected_scale_shape:
|
| 424 |
+
raise ValueError(
|
| 425 |
+
f"INT8 module {module_path}: scale shape must be {expected_scale_shape}, got {tuple(scale.shape)}"
|
| 426 |
+
)
|
| 427 |
+
|
| 428 |
+
|
| 429 |
+
# endregion
|
| 430 |
+
|
| 431 |
+
|
| 432 |
+
# region monkey patch
|
| 433 |
+
|
| 434 |
+
|
| 435 |
+
def nvfp4_linear_forward_patch(self: nn.Linear, x: torch.Tensor) -> torch.Tensor:
|
| 436 |
+
pre_quant_scale = getattr(self, "pre_quant_scale", None)
|
| 437 |
+
if pre_quant_scale is not None:
|
| 438 |
+
x = x * pre_quant_scale
|
| 439 |
+
if self._nvfp4_use_scaled_mm:
|
| 440 |
+
x_2d = x.reshape(-1, x.shape[-1])
|
| 441 |
+
out = nvfp4_scaled_mm_linear(
|
| 442 |
+
x_2d, self.weight, self.nvfp4_block_scale, self.nvfp4_scale, self.bias, self._nvfp4_orig_shape[0]
|
| 443 |
+
)
|
| 444 |
+
return out.reshape(*x.shape[:-1], out.shape[-1])
|
| 445 |
+
weight = dequantize_nvfp4(self.weight, self.nvfp4_block_scale, self.nvfp4_scale, self._nvfp4_orig_shape, x.dtype)
|
| 446 |
+
return F.linear(x, weight, self.bias)
|
| 447 |
+
|
| 448 |
+
|
| 449 |
+
def int8_embedding_forward_patch(self: nn.Embedding, input: torch.Tensor) -> torch.Tensor:
|
| 450 |
+
rows = self.weight[input] # index_select works on int8; padding_idx etc. only affect training
|
| 451 |
+
return (rows.float() * self.scale_weight[input]).to(self._int8_dequant_dtype)
|
| 452 |
+
|
| 453 |
+
|
| 454 |
+
def apply_nvfp4_monkey_patch(
|
| 455 |
+
model: nn.Module,
|
| 456 |
+
optimized_state_dict: dict,
|
| 457 |
+
nvfp4_module_shapes: Dict[str, Tuple[int, int]],
|
| 458 |
+
int8_embedding_modules: List[str],
|
| 459 |
+
use_scaled_mm: bool = False,
|
| 460 |
+
embedding_dtype: torch.dtype = torch.bfloat16,
|
| 461 |
+
) -> nn.Module:
|
| 462 |
+
"""Patch NVFP4 Linear and INT8 embedding modules so a strict assign load can follow.
|
| 463 |
+
|
| 464 |
+
The patched modules get placeholder parameters/buffers with the quantized shapes and
|
| 465 |
+
dtypes (on the meta device); ``model.load_state_dict(state_dict, strict=True,
|
| 466 |
+
assign=True)`` then installs the real tensors. Modules stay ``nn.Linear`` /
|
| 467 |
+
``nn.Embedding`` (patched forward), mirroring the ConvRot INT8 approach.
|
| 468 |
+
"""
|
| 469 |
+
if use_scaled_mm and not nvfp4_scaled_mm_available():
|
| 470 |
+
raise ValueError(
|
| 471 |
+
"NVFP4 scaled_mm requires PyTorch 2.10+ (torch.float4_e2m1fn_x2 and torch.nn.functional.scaled_mm)."
|
| 472 |
+
" Omit the scaled_mm option to use the dequantize fallback."
|
| 473 |
+
" / NVFP4 scaled_mm には PyTorch 2.10 以降が必要です。scaled_mm オプションを外すと dequantize フォールバックで動作します。"
|
| 474 |
+
)
|
| 475 |
+
|
| 476 |
+
modules_by_name = dict(model.named_modules())
|
| 477 |
+
patched_count = 0
|
| 478 |
+
|
| 479 |
+
for name, (out_features, in_features) in nvfp4_module_shapes.items():
|
| 480 |
+
module = modules_by_name.get(name)
|
| 481 |
+
if not isinstance(module, nn.Linear):
|
| 482 |
+
raise ValueError(f"NVFP4 state dict declares {name}, which is not an nn.Linear in the model")
|
| 483 |
+
weight_key = name + ".weight"
|
| 484 |
+
module.weight = nn.Parameter(torch.empty_like(optimized_state_dict[weight_key], device="meta"), requires_grad=False)
|
| 485 |
+
module.register_buffer(
|
| 486 |
+
"nvfp4_block_scale", torch.empty_like(optimized_state_dict[name + ".nvfp4_block_scale"], device="meta")
|
| 487 |
+
)
|
| 488 |
+
module.register_buffer("nvfp4_scale", torch.empty((), dtype=torch.float32, device="meta"))
|
| 489 |
+
pre_quant_key = name + COMFY_PRE_QUANT_SCALE_SUFFIX
|
| 490 |
+
if pre_quant_key in optimized_state_dict:
|
| 491 |
+
module.register_buffer("pre_quant_scale", torch.empty_like(optimized_state_dict[pre_quant_key], device="meta"))
|
| 492 |
+
module._nvfp4_orig_shape = (out_features, in_features)
|
| 493 |
+
module._nvfp4_use_scaled_mm = use_scaled_mm
|
| 494 |
+
module.forward = nvfp4_linear_forward_patch.__get__(module, type(module))
|
| 495 |
+
patched_count += 1
|
| 496 |
+
|
| 497 |
+
for name in int8_embedding_modules:
|
| 498 |
+
module = modules_by_name.get(name)
|
| 499 |
+
if not isinstance(module, nn.Embedding):
|
| 500 |
+
raise ValueError(
|
| 501 |
+
f"int8_tensorwise module {name} is not an nn.Embedding; INT8 per-row quantization is only"
|
| 502 |
+
" supported for embeddings (use ConvRot INT8 checkpoints for Linear layers)"
|
| 503 |
+
)
|
| 504 |
+
module.weight = nn.Parameter(torch.empty_like(optimized_state_dict[name + ".weight"], device="meta"), requires_grad=False)
|
| 505 |
+
module.register_buffer("scale_weight", torch.empty_like(optimized_state_dict[name + ".scale_weight"], device="meta"))
|
| 506 |
+
module._int8_dequant_dtype = embedding_dtype
|
| 507 |
+
module.forward = int8_embedding_forward_patch.__get__(module, type(module))
|
| 508 |
+
patched_count += 1
|
| 509 |
+
|
| 510 |
+
if not use_scaled_mm and nvfp4_module_shapes:
|
| 511 |
+
logger.info("NVFP4 runs in weight-only mode (transient dequantization per forward)")
|
| 512 |
+
model.is_nvfp4 = True
|
| 513 |
+
model.nvfp4_layer_count = len(nvfp4_module_shapes)
|
| 514 |
+
logger.info(f"Number of NVFP4/INT8 monkey-patched modules: {patched_count}")
|
| 515 |
+
return model
|
| 516 |
+
|
| 517 |
+
|
| 518 |
+
# endregion
|
third_party/musubi_tuner/networks/__init__.py
ADDED
|
File without changes
|
third_party/musubi_tuner/networks/lora.py
ADDED
|
@@ -0,0 +1,1024 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# LoRA network module: currently conv2d is not fully supported
|
| 2 |
+
# reference:
|
| 3 |
+
# https://github.com/microsoft/LoRA/blob/main/loralib/layers.py
|
| 4 |
+
# https://github.com/cloneofsimo/lora/blob/master/lora_diffusion/lora.py
|
| 5 |
+
|
| 6 |
+
import ast
|
| 7 |
+
import math
|
| 8 |
+
import os
|
| 9 |
+
import re
|
| 10 |
+
from typing import Any, Dict, List, Optional, Type, Union
|
| 11 |
+
from transformers import CLIPTextModel
|
| 12 |
+
import torch
|
| 13 |
+
import torch.nn as nn
|
| 14 |
+
|
| 15 |
+
import logging
|
| 16 |
+
|
| 17 |
+
logger = logging.getLogger(__name__)
|
| 18 |
+
logging.basicConfig(level=logging.INFO)
|
| 19 |
+
|
| 20 |
+
HUNYUAN_TARGET_REPLACE_MODULES = ["MMDoubleStreamBlock", "MMSingleStreamBlock"]
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
class LoRAModule(torch.nn.Module):
|
| 24 |
+
"""
|
| 25 |
+
replaces forward method of the original Linear, instead of replacing the original Linear module.
|
| 26 |
+
"""
|
| 27 |
+
|
| 28 |
+
def __init__(
|
| 29 |
+
self,
|
| 30 |
+
lora_name,
|
| 31 |
+
org_module: torch.nn.Module,
|
| 32 |
+
multiplier=1.0,
|
| 33 |
+
lora_dim=4,
|
| 34 |
+
alpha=1,
|
| 35 |
+
dropout=None,
|
| 36 |
+
rank_dropout=None,
|
| 37 |
+
module_dropout=None,
|
| 38 |
+
split_dims: Optional[List[int]] = None,
|
| 39 |
+
):
|
| 40 |
+
"""
|
| 41 |
+
if alpha == 0 or None, alpha is rank (no scaling).
|
| 42 |
+
|
| 43 |
+
split_dims is used to mimic the split qkv of multi-head attention.
|
| 44 |
+
"""
|
| 45 |
+
super().__init__()
|
| 46 |
+
self.lora_name = lora_name
|
| 47 |
+
|
| 48 |
+
if org_module.__class__.__name__ in ("Conv2d", "Conv3d"):
|
| 49 |
+
in_dim = org_module.in_channels
|
| 50 |
+
out_dim = org_module.out_channels
|
| 51 |
+
else:
|
| 52 |
+
in_dim = org_module.in_features
|
| 53 |
+
out_dim = org_module.out_features
|
| 54 |
+
|
| 55 |
+
self.lora_dim = lora_dim
|
| 56 |
+
self.split_dims = split_dims
|
| 57 |
+
|
| 58 |
+
if split_dims is None:
|
| 59 |
+
if org_module.__class__.__name__ == "Conv2d":
|
| 60 |
+
kernel_size = org_module.kernel_size
|
| 61 |
+
stride = org_module.stride
|
| 62 |
+
padding = org_module.padding
|
| 63 |
+
self.lora_down = torch.nn.Conv2d(in_dim, self.lora_dim, kernel_size, stride, padding, bias=False)
|
| 64 |
+
self.lora_up = torch.nn.Conv2d(self.lora_dim, out_dim, (1, 1), (1, 1), bias=False)
|
| 65 |
+
elif org_module.__class__.__name__ == "Conv3d":
|
| 66 |
+
kernel_size = org_module.kernel_size
|
| 67 |
+
stride = org_module.stride
|
| 68 |
+
padding = org_module.padding
|
| 69 |
+
self.lora_down = torch.nn.Conv3d(in_dim, self.lora_dim, kernel_size, stride, padding, bias=False)
|
| 70 |
+
self.lora_up = torch.nn.Conv3d(self.lora_dim, out_dim, (1, 1, 1), (1, 1, 1), bias=False)
|
| 71 |
+
else:
|
| 72 |
+
self.lora_down = torch.nn.Linear(in_dim, self.lora_dim, bias=False)
|
| 73 |
+
self.lora_up = torch.nn.Linear(self.lora_dim, out_dim, bias=False)
|
| 74 |
+
|
| 75 |
+
torch.nn.init.kaiming_uniform_(self.lora_down.weight, a=math.sqrt(5))
|
| 76 |
+
torch.nn.init.zeros_(self.lora_up.weight)
|
| 77 |
+
else:
|
| 78 |
+
# conv2d not supported
|
| 79 |
+
assert sum(split_dims) == out_dim, "sum of split_dims must be equal to out_dim"
|
| 80 |
+
assert org_module.__class__.__name__ == "Linear", "split_dims is only supported for Linear"
|
| 81 |
+
# print(f"split_dims: {split_dims}")
|
| 82 |
+
self.lora_down = torch.nn.ModuleList(
|
| 83 |
+
[torch.nn.Linear(in_dim, self.lora_dim, bias=False) for _ in range(len(split_dims))]
|
| 84 |
+
)
|
| 85 |
+
self.lora_up = torch.nn.ModuleList([torch.nn.Linear(self.lora_dim, split_dim, bias=False) for split_dim in split_dims])
|
| 86 |
+
for lora_down in self.lora_down:
|
| 87 |
+
torch.nn.init.kaiming_uniform_(lora_down.weight, a=math.sqrt(5))
|
| 88 |
+
for lora_up in self.lora_up:
|
| 89 |
+
torch.nn.init.zeros_(lora_up.weight)
|
| 90 |
+
|
| 91 |
+
if type(alpha) == torch.Tensor:
|
| 92 |
+
alpha = alpha.detach().float().cpu().item() # without casting, bf16 causes error
|
| 93 |
+
alpha = self.lora_dim if alpha is None or alpha == 0 else alpha
|
| 94 |
+
self.scale = alpha / self.lora_dim
|
| 95 |
+
self.register_buffer("alpha", torch.tensor(alpha)) # for save/load
|
| 96 |
+
|
| 97 |
+
# same as microsoft's
|
| 98 |
+
self.multiplier = multiplier
|
| 99 |
+
self.org_module = org_module # remove in applying
|
| 100 |
+
self.dropout = dropout
|
| 101 |
+
self.rank_dropout = rank_dropout
|
| 102 |
+
self.module_dropout = module_dropout
|
| 103 |
+
# honored by the training forward too, so LoRANetwork.set_enabled(False) yields the
|
| 104 |
+
# frozen base everywhere (e.g. the MiniMax-H3 teacher-matching forward)
|
| 105 |
+
self.enabled = True
|
| 106 |
+
|
| 107 |
+
# Used by _rank_dropout_mask to locate the rank dim in lx; saved now since
|
| 108 |
+
# org_module is deleted in apply_to() before forward() ever runs.
|
| 109 |
+
self._rank_dropout_is_conv = org_module.__class__.__name__ in ("Conv2d", "Conv3d")
|
| 110 |
+
|
| 111 |
+
def _autocast_enabled_for(self, x):
|
| 112 |
+
if not x.is_floating_point():
|
| 113 |
+
return False
|
| 114 |
+
try:
|
| 115 |
+
return torch.is_autocast_enabled(x.device.type)
|
| 116 |
+
except TypeError:
|
| 117 |
+
return torch.is_autocast_enabled()
|
| 118 |
+
|
| 119 |
+
def _rank_dropout_mask(self, lx: torch.Tensor) -> torch.Tensor:
|
| 120 |
+
"""Build a per-rank keep/drop mask shaped to broadcast against lx.
|
| 121 |
+
|
| 122 |
+
Where the rank dim sits in lx depends on the wrapped module: Conv puts it at
|
| 123 |
+
dim=1 (channels), Linear always puts it last, no matter how many leading
|
| 124 |
+
dims lx has (e.g. a 4D (b, l, d, rank) input, as in krea2's text-fusion
|
| 125 |
+
projector).
|
| 126 |
+
"""
|
| 127 |
+
mask = torch.rand((lx.size(0), self.lora_dim), device=lx.device) > self.rank_dropout
|
| 128 |
+
extra_dims = lx.dim() - 2
|
| 129 |
+
if extra_dims <= 0:
|
| 130 |
+
return mask
|
| 131 |
+
if self._rank_dropout_is_conv:
|
| 132 |
+
return mask.view(mask.shape[0], mask.shape[1], *([1] * extra_dims))
|
| 133 |
+
else:
|
| 134 |
+
return mask.view(mask.shape[0], *([1] * extra_dims), mask.shape[1])
|
| 135 |
+
|
| 136 |
+
def _lora_input(self, x):
|
| 137 |
+
if self.split_dims is None:
|
| 138 |
+
target_dtype = self.lora_down.weight.dtype
|
| 139 |
+
else:
|
| 140 |
+
target_dtype = self.lora_down[0].weight.dtype
|
| 141 |
+
if x.is_floating_point() and x.dtype != target_dtype and not self._autocast_enabled_for(x):
|
| 142 |
+
return x.to(target_dtype)
|
| 143 |
+
return x
|
| 144 |
+
|
| 145 |
+
def _match_org_dtype(self, value, org_forwarded):
|
| 146 |
+
"""Round ``value`` to the base output dtype.
|
| 147 |
+
|
| 148 |
+
Used to bring the LoRA-augmented output back to ``org_forwarded``'s dtype in the
|
| 149 |
+
autocast-free mixed-dtype regime (e.g. fp32 LoRA on a bf16 base). Callers pass the
|
| 150 |
+
full ``org_forwarded + delta`` sum here so the (possibly higher-precision) delta is
|
| 151 |
+
kept through the addition and the result is rounded only once. A no-op when dtypes
|
| 152 |
+
already match (autocast-on / all-fp32 regimes)."""
|
| 153 |
+
if value.is_floating_point() and org_forwarded.is_floating_point() and value.dtype != org_forwarded.dtype:
|
| 154 |
+
return value.to(org_forwarded.dtype)
|
| 155 |
+
return value
|
| 156 |
+
|
| 157 |
+
def apply_to(self):
|
| 158 |
+
self.org_forward = self.org_module.forward
|
| 159 |
+
self.org_module.forward = self.forward
|
| 160 |
+
del self.org_module
|
| 161 |
+
|
| 162 |
+
def forward(self, x):
|
| 163 |
+
if not self.enabled:
|
| 164 |
+
return self.org_forward(x)
|
| 165 |
+
|
| 166 |
+
org_forwarded = self.org_forward(x)
|
| 167 |
+
|
| 168 |
+
# module dropout
|
| 169 |
+
if self.module_dropout is not None and self.training:
|
| 170 |
+
if torch.rand(1) < self.module_dropout:
|
| 171 |
+
return org_forwarded
|
| 172 |
+
|
| 173 |
+
lora_input = self._lora_input(x)
|
| 174 |
+
if self.split_dims is None:
|
| 175 |
+
lx = self.lora_down(lora_input)
|
| 176 |
+
|
| 177 |
+
# normal dropout
|
| 178 |
+
if self.dropout is not None and self.training:
|
| 179 |
+
lx = torch.nn.functional.dropout(lx, p=self.dropout)
|
| 180 |
+
|
| 181 |
+
# rank dropout
|
| 182 |
+
if self.rank_dropout is not None and self.training:
|
| 183 |
+
lx = lx * self._rank_dropout_mask(lx)
|
| 184 |
+
|
| 185 |
+
# scaling for rank dropout: treat as if the rank is changed
|
| 186 |
+
scale = self.scale * (1.0 / (1.0 - self.rank_dropout)) # redundant for readability
|
| 187 |
+
else:
|
| 188 |
+
scale = self.scale
|
| 189 |
+
|
| 190 |
+
lx = self.lora_up(lx)
|
| 191 |
+
|
| 192 |
+
# Add in the (possibly higher-precision) delta dtype, then round the sum once.
|
| 193 |
+
return self._match_org_dtype(org_forwarded + lx * self.multiplier * scale, org_forwarded)
|
| 194 |
+
else:
|
| 195 |
+
lxs = [lora_down(lora_input) for lora_down in self.lora_down]
|
| 196 |
+
|
| 197 |
+
# normal dropout
|
| 198 |
+
if self.dropout is not None and self.training:
|
| 199 |
+
lxs = [torch.nn.functional.dropout(lx, p=self.dropout) for lx in lxs]
|
| 200 |
+
|
| 201 |
+
# rank dropout
|
| 202 |
+
if self.rank_dropout is not None and self.training:
|
| 203 |
+
lxs = [lx * self._rank_dropout_mask(lx) for lx in lxs]
|
| 204 |
+
|
| 205 |
+
# scaling for rank dropout: treat as if the rank is changed
|
| 206 |
+
scale = self.scale * (1.0 / (1.0 - self.rank_dropout)) # redundant for readability
|
| 207 |
+
else:
|
| 208 |
+
scale = self.scale
|
| 209 |
+
|
| 210 |
+
lxs = [lora_up(lx) for lora_up, lx in zip(self.lora_up, lxs)]
|
| 211 |
+
|
| 212 |
+
# Add in the (possibly higher-precision) delta dtype, then round the sum once.
|
| 213 |
+
return self._match_org_dtype(org_forwarded + torch.cat(lxs, dim=-1) * self.multiplier * scale, org_forwarded)
|
| 214 |
+
|
| 215 |
+
|
| 216 |
+
class LoRAInfModule(LoRAModule):
|
| 217 |
+
def __init__(
|
| 218 |
+
self,
|
| 219 |
+
lora_name,
|
| 220 |
+
org_module: torch.nn.Module,
|
| 221 |
+
multiplier=1.0,
|
| 222 |
+
lora_dim=4,
|
| 223 |
+
alpha=1,
|
| 224 |
+
**kwargs,
|
| 225 |
+
):
|
| 226 |
+
# no dropout for inference
|
| 227 |
+
super().__init__(lora_name, org_module, multiplier, lora_dim, alpha)
|
| 228 |
+
|
| 229 |
+
self.org_module_ref = [org_module] # for reference
|
| 230 |
+
self.network: LoRANetwork = None
|
| 231 |
+
|
| 232 |
+
def set_network(self, network):
|
| 233 |
+
self.network = network
|
| 234 |
+
|
| 235 |
+
def merge_to(self, sd, dtype, device, non_blocking=False):
|
| 236 |
+
# extract weight from org_module
|
| 237 |
+
org_sd = self.org_module.state_dict()
|
| 238 |
+
weight = org_sd["weight"]
|
| 239 |
+
org_dtype = weight.dtype
|
| 240 |
+
org_device = weight.device
|
| 241 |
+
weight = weight.to(device, dtype=torch.float, non_blocking=non_blocking) # for calculation
|
| 242 |
+
|
| 243 |
+
if dtype is None:
|
| 244 |
+
dtype = org_dtype
|
| 245 |
+
if device is None:
|
| 246 |
+
device = org_device
|
| 247 |
+
|
| 248 |
+
if self.split_dims is None:
|
| 249 |
+
# get up/down weight
|
| 250 |
+
down_weight = sd["lora_down.weight"].to(device, dtype=torch.float, non_blocking=non_blocking)
|
| 251 |
+
up_weight = sd["lora_up.weight"].to(device, dtype=torch.float, non_blocking=non_blocking)
|
| 252 |
+
|
| 253 |
+
# merge weight
|
| 254 |
+
if len(weight.size()) == 2:
|
| 255 |
+
# linear
|
| 256 |
+
weight = weight + self.multiplier * (up_weight @ down_weight) * self.scale
|
| 257 |
+
elif len(weight.size()) == 4:
|
| 258 |
+
if down_weight.size()[2:4] == (1, 1):
|
| 259 |
+
# conv2d 1x1
|
| 260 |
+
weight = (
|
| 261 |
+
weight
|
| 262 |
+
+ self.multiplier
|
| 263 |
+
* (up_weight.squeeze(3).squeeze(2) @ down_weight.squeeze(3).squeeze(2)).unsqueeze(2).unsqueeze(3)
|
| 264 |
+
* self.scale
|
| 265 |
+
)
|
| 266 |
+
else:
|
| 267 |
+
# conv2d 3x3
|
| 268 |
+
conved = torch.nn.functional.conv2d(down_weight.permute(1, 0, 2, 3), up_weight).permute(1, 0, 2, 3)
|
| 269 |
+
# logger.info(conved.size(), weight.size(), module.stride, module.padding)
|
| 270 |
+
weight = weight + self.multiplier * conved * self.scale
|
| 271 |
+
elif len(weight.size()) == 5:
|
| 272 |
+
if down_weight.size()[2:5] == (1, 1, 1):
|
| 273 |
+
# conv3d 1x1x1
|
| 274 |
+
weight = (
|
| 275 |
+
weight
|
| 276 |
+
+ self.multiplier
|
| 277 |
+
* (up_weight.squeeze(4).squeeze(3).squeeze(2) @ down_weight.squeeze(4).squeeze(3).squeeze(2))
|
| 278 |
+
.unsqueeze(2)
|
| 279 |
+
.unsqueeze(3)
|
| 280 |
+
.unsqueeze(4)
|
| 281 |
+
* self.scale
|
| 282 |
+
)
|
| 283 |
+
else:
|
| 284 |
+
conved = torch.nn.functional.conv3d(down_weight.permute(1, 0, 2, 3, 4), up_weight).permute(1, 0, 2, 3, 4)
|
| 285 |
+
weight = weight + self.multiplier * conved * self.scale
|
| 286 |
+
else:
|
| 287 |
+
raise ValueError(f"Unsupported LoRA target weight shape: {weight.size()}")
|
| 288 |
+
|
| 289 |
+
# set weight to org_module
|
| 290 |
+
org_sd["weight"] = weight.to(org_device, dtype=dtype) # back to CPU without non_blocking
|
| 291 |
+
self.org_module.load_state_dict(org_sd)
|
| 292 |
+
else:
|
| 293 |
+
# split_dims
|
| 294 |
+
total_dims = sum(self.split_dims)
|
| 295 |
+
for i in range(len(self.split_dims)):
|
| 296 |
+
# get up/down weight
|
| 297 |
+
down_weight = sd[f"lora_down.{i}.weight"].to(device, torch.float, non_blocking=non_blocking) # (rank, in_dim)
|
| 298 |
+
up_weight = sd[f"lora_up.{i}.weight"].to(device, torch.float, non_blocking=non_blocking) # (split dim, rank)
|
| 299 |
+
|
| 300 |
+
# pad up_weight -> (total_dims, rank)
|
| 301 |
+
padded_up_weight = torch.zeros((total_dims, up_weight.size(0)), device=device, dtype=torch.float)
|
| 302 |
+
padded_up_weight[sum(self.split_dims[:i]) : sum(self.split_dims[: i + 1])] = up_weight
|
| 303 |
+
|
| 304 |
+
# merge weight
|
| 305 |
+
weight = weight + self.multiplier * (up_weight @ down_weight) * self.scale
|
| 306 |
+
|
| 307 |
+
# set weight to org_module
|
| 308 |
+
org_sd["weight"] = weight.to(org_device, dtype) # back to CPU without non_blocking
|
| 309 |
+
self.org_module.load_state_dict(org_sd)
|
| 310 |
+
|
| 311 |
+
# return weight for merge
|
| 312 |
+
def get_weight(self, multiplier=None):
|
| 313 |
+
if multiplier is None:
|
| 314 |
+
multiplier = self.multiplier
|
| 315 |
+
|
| 316 |
+
# get up/down weight from module
|
| 317 |
+
up_weight = self.lora_up.weight.to(torch.float)
|
| 318 |
+
down_weight = self.lora_down.weight.to(torch.float)
|
| 319 |
+
|
| 320 |
+
# pre-calculated weight
|
| 321 |
+
if len(down_weight.size()) == 2:
|
| 322 |
+
# linear
|
| 323 |
+
weight = self.multiplier * (up_weight @ down_weight) * self.scale
|
| 324 |
+
elif len(down_weight.size()) == 4:
|
| 325 |
+
if down_weight.size()[2:4] == (1, 1):
|
| 326 |
+
# conv2d 1x1
|
| 327 |
+
weight = (
|
| 328 |
+
self.multiplier
|
| 329 |
+
* (up_weight.squeeze(3).squeeze(2) @ down_weight.squeeze(3).squeeze(2)).unsqueeze(2).unsqueeze(3)
|
| 330 |
+
* self.scale
|
| 331 |
+
)
|
| 332 |
+
else:
|
| 333 |
+
# conv2d 3x3
|
| 334 |
+
conved = torch.nn.functional.conv2d(down_weight.permute(1, 0, 2, 3), up_weight).permute(1, 0, 2, 3)
|
| 335 |
+
weight = self.multiplier * conved * self.scale
|
| 336 |
+
elif len(down_weight.size()) == 5:
|
| 337 |
+
if down_weight.size()[2:5] == (1, 1, 1):
|
| 338 |
+
# conv3d 1x1x1
|
| 339 |
+
weight = (
|
| 340 |
+
self.multiplier
|
| 341 |
+
* (up_weight.squeeze(4).squeeze(3).squeeze(2) @ down_weight.squeeze(4).squeeze(3).squeeze(2))
|
| 342 |
+
.unsqueeze(2)
|
| 343 |
+
.unsqueeze(3)
|
| 344 |
+
.unsqueeze(4)
|
| 345 |
+
* self.scale
|
| 346 |
+
)
|
| 347 |
+
else:
|
| 348 |
+
conved = torch.nn.functional.conv3d(down_weight.permute(1, 0, 2, 3, 4), up_weight).permute(1, 0, 2, 3, 4)
|
| 349 |
+
weight = self.multiplier * conved * self.scale
|
| 350 |
+
else:
|
| 351 |
+
raise ValueError(f"Unsupported LoRA weight shape: {down_weight.size()}")
|
| 352 |
+
|
| 353 |
+
return weight
|
| 354 |
+
|
| 355 |
+
def default_forward(self, x):
|
| 356 |
+
# logger.info(f"default_forward {self.lora_name} {x.size()}")
|
| 357 |
+
lora_input = self._lora_input(x)
|
| 358 |
+
if self.split_dims is None:
|
| 359 |
+
lx = self.lora_down(lora_input)
|
| 360 |
+
lx = self.lora_up(lx)
|
| 361 |
+
org_forwarded = self.org_forward(x)
|
| 362 |
+
# Add in the (possibly higher-precision) delta dtype, then round the sum once.
|
| 363 |
+
return self._match_org_dtype(org_forwarded + lx * self.multiplier * self.scale, org_forwarded)
|
| 364 |
+
else:
|
| 365 |
+
lxs = [lora_down(lora_input) for lora_down in self.lora_down]
|
| 366 |
+
lxs = [lora_up(lx) for lora_up, lx in zip(self.lora_up, lxs)]
|
| 367 |
+
org_forwarded = self.org_forward(x)
|
| 368 |
+
# Add in the (possibly higher-precision) delta dtype, then round the sum once.
|
| 369 |
+
return self._match_org_dtype(org_forwarded + torch.cat(lxs, dim=-1) * self.multiplier * self.scale, org_forwarded)
|
| 370 |
+
|
| 371 |
+
def forward(self, x):
|
| 372 |
+
if not self.enabled:
|
| 373 |
+
return self.org_forward(x)
|
| 374 |
+
return self.default_forward(x)
|
| 375 |
+
|
| 376 |
+
|
| 377 |
+
def create_arch_network(
|
| 378 |
+
multiplier: float,
|
| 379 |
+
network_dim: Optional[int],
|
| 380 |
+
network_alpha: Optional[float],
|
| 381 |
+
vae: nn.Module,
|
| 382 |
+
text_encoders: List[nn.Module],
|
| 383 |
+
unet: nn.Module,
|
| 384 |
+
neuron_dropout: Optional[float] = None,
|
| 385 |
+
**kwargs,
|
| 386 |
+
):
|
| 387 |
+
# add default exclude patterns
|
| 388 |
+
exclude_patterns = kwargs.get("exclude_patterns", None)
|
| 389 |
+
if exclude_patterns is None:
|
| 390 |
+
exclude_patterns = []
|
| 391 |
+
else:
|
| 392 |
+
exclude_patterns = ast.literal_eval(exclude_patterns)
|
| 393 |
+
|
| 394 |
+
# exclude if 'img_mod', 'txt_mod' or 'modulation' in the name
|
| 395 |
+
exclude_patterns.append(r".*(img_mod|txt_mod|modulation).*")
|
| 396 |
+
|
| 397 |
+
kwargs["exclude_patterns"] = exclude_patterns
|
| 398 |
+
|
| 399 |
+
return create_network(
|
| 400 |
+
HUNYUAN_TARGET_REPLACE_MODULES,
|
| 401 |
+
"lora_unet",
|
| 402 |
+
multiplier,
|
| 403 |
+
network_dim,
|
| 404 |
+
network_alpha,
|
| 405 |
+
vae,
|
| 406 |
+
text_encoders,
|
| 407 |
+
unet,
|
| 408 |
+
neuron_dropout=neuron_dropout,
|
| 409 |
+
**kwargs,
|
| 410 |
+
)
|
| 411 |
+
|
| 412 |
+
|
| 413 |
+
def create_network(
|
| 414 |
+
target_replace_modules: List[str],
|
| 415 |
+
prefix: str,
|
| 416 |
+
multiplier: float,
|
| 417 |
+
network_dim: Optional[int],
|
| 418 |
+
network_alpha: Optional[float],
|
| 419 |
+
vae: nn.Module,
|
| 420 |
+
text_encoders: List[nn.Module],
|
| 421 |
+
unet: nn.Module,
|
| 422 |
+
neuron_dropout: Optional[float] = None,
|
| 423 |
+
module_class: Type[object] = None,
|
| 424 |
+
module_kwargs: Optional[Dict[str, Any]] = None,
|
| 425 |
+
**kwargs,
|
| 426 |
+
):
|
| 427 |
+
"""architecture independent network creation"""
|
| 428 |
+
if network_dim is None:
|
| 429 |
+
network_dim = 4 # default
|
| 430 |
+
if network_alpha is None:
|
| 431 |
+
network_alpha = 1.0
|
| 432 |
+
|
| 433 |
+
# extract dim/alpha for conv2d, and block dim
|
| 434 |
+
conv_dim = kwargs.get("conv_dim", None)
|
| 435 |
+
conv_alpha = kwargs.get("conv_alpha", None)
|
| 436 |
+
if conv_dim is not None:
|
| 437 |
+
conv_dim = int(conv_dim)
|
| 438 |
+
if conv_alpha is None:
|
| 439 |
+
conv_alpha = 1.0
|
| 440 |
+
else:
|
| 441 |
+
conv_alpha = float(conv_alpha)
|
| 442 |
+
|
| 443 |
+
# TODO generic rank/dim setting with regular expression
|
| 444 |
+
|
| 445 |
+
# rank/module dropout
|
| 446 |
+
rank_dropout = kwargs.get("rank_dropout", None)
|
| 447 |
+
if rank_dropout is not None:
|
| 448 |
+
rank_dropout = float(rank_dropout)
|
| 449 |
+
module_dropout = kwargs.get("module_dropout", None)
|
| 450 |
+
if module_dropout is not None:
|
| 451 |
+
module_dropout = float(module_dropout)
|
| 452 |
+
|
| 453 |
+
# verbose
|
| 454 |
+
verbose = kwargs.get("verbose", False)
|
| 455 |
+
if verbose is not None:
|
| 456 |
+
verbose = True if verbose == "True" else False
|
| 457 |
+
|
| 458 |
+
# regular expression for module selection: exclude and include
|
| 459 |
+
exclude_patterns = kwargs.get("exclude_patterns", None)
|
| 460 |
+
if exclude_patterns is not None and isinstance(exclude_patterns, str):
|
| 461 |
+
exclude_patterns = ast.literal_eval(exclude_patterns)
|
| 462 |
+
include_patterns = kwargs.get("include_patterns", None)
|
| 463 |
+
if include_patterns is not None and isinstance(include_patterns, str):
|
| 464 |
+
include_patterns = ast.literal_eval(include_patterns)
|
| 465 |
+
|
| 466 |
+
if module_class is None:
|
| 467 |
+
module_class = LoRAModule
|
| 468 |
+
|
| 469 |
+
# too many arguments ( ^ω^)・・・
|
| 470 |
+
network = LoRANetwork(
|
| 471 |
+
target_replace_modules,
|
| 472 |
+
prefix,
|
| 473 |
+
text_encoders,
|
| 474 |
+
unet,
|
| 475 |
+
multiplier=multiplier,
|
| 476 |
+
lora_dim=network_dim,
|
| 477 |
+
alpha=network_alpha,
|
| 478 |
+
dropout=neuron_dropout,
|
| 479 |
+
rank_dropout=rank_dropout,
|
| 480 |
+
module_dropout=module_dropout,
|
| 481 |
+
conv_lora_dim=conv_dim,
|
| 482 |
+
conv_alpha=conv_alpha,
|
| 483 |
+
module_class=module_class,
|
| 484 |
+
module_kwargs=module_kwargs,
|
| 485 |
+
exclude_patterns=exclude_patterns,
|
| 486 |
+
include_patterns=include_patterns,
|
| 487 |
+
verbose=verbose,
|
| 488 |
+
)
|
| 489 |
+
|
| 490 |
+
loraplus_lr_ratio = kwargs.get("loraplus_lr_ratio", None)
|
| 491 |
+
# loraplus_unet_lr_ratio = kwargs.get("loraplus_unet_lr_ratio", None)
|
| 492 |
+
# loraplus_text_encoder_lr_ratio = kwargs.get("loraplus_text_encoder_lr_ratio", None)
|
| 493 |
+
loraplus_lr_ratio = float(loraplus_lr_ratio) if loraplus_lr_ratio is not None else None
|
| 494 |
+
# loraplus_unet_lr_ratio = float(loraplus_unet_lr_ratio) if loraplus_unet_lr_ratio is not None else None
|
| 495 |
+
# loraplus_text_encoder_lr_ratio = float(loraplus_text_encoder_lr_ratio) if loraplus_text_encoder_lr_ratio is not None else None
|
| 496 |
+
if loraplus_lr_ratio is not None: # or loraplus_unet_lr_ratio is not None or loraplus_text_encoder_lr_ratio is not None:
|
| 497 |
+
network.set_loraplus_lr_ratio(loraplus_lr_ratio) # , loraplus_unet_lr_ratio, loraplus_text_encoder_lr_ratio)
|
| 498 |
+
|
| 499 |
+
return network
|
| 500 |
+
|
| 501 |
+
|
| 502 |
+
class LoRANetwork(torch.nn.Module):
|
| 503 |
+
# only supports U-Net (DiT), Text Encoders are not supported
|
| 504 |
+
|
| 505 |
+
def __init__(
|
| 506 |
+
self,
|
| 507 |
+
target_replace_modules: List[str],
|
| 508 |
+
prefix: str,
|
| 509 |
+
text_encoders: Union[List[CLIPTextModel], CLIPTextModel],
|
| 510 |
+
unet: nn.Module,
|
| 511 |
+
multiplier: float = 1.0,
|
| 512 |
+
lora_dim: int = 4,
|
| 513 |
+
alpha: float = 1,
|
| 514 |
+
dropout: Optional[float] = None,
|
| 515 |
+
rank_dropout: Optional[float] = None,
|
| 516 |
+
module_dropout: Optional[float] = None,
|
| 517 |
+
conv_lora_dim: Optional[int] = None,
|
| 518 |
+
conv_alpha: Optional[float] = None,
|
| 519 |
+
module_class: Type[object] = LoRAModule,
|
| 520 |
+
module_kwargs: Optional[Dict[str, Any]] = None,
|
| 521 |
+
modules_dim: Optional[Dict[str, int]] = None,
|
| 522 |
+
modules_alpha: Optional[Dict[str, int]] = None,
|
| 523 |
+
exclude_patterns: Optional[List[str]] = None,
|
| 524 |
+
include_patterns: Optional[List[str]] = None,
|
| 525 |
+
verbose: Optional[bool] = False,
|
| 526 |
+
) -> None:
|
| 527 |
+
super().__init__()
|
| 528 |
+
self.multiplier = multiplier
|
| 529 |
+
|
| 530 |
+
self.lora_dim = lora_dim
|
| 531 |
+
self.alpha = alpha
|
| 532 |
+
self.conv_lora_dim = conv_lora_dim
|
| 533 |
+
self.conv_alpha = conv_alpha
|
| 534 |
+
self.dropout = dropout
|
| 535 |
+
self.rank_dropout = rank_dropout
|
| 536 |
+
self.module_dropout = module_dropout
|
| 537 |
+
self.target_replace_modules = target_replace_modules
|
| 538 |
+
self.prefix = prefix
|
| 539 |
+
self.module_kwargs = module_kwargs or {}
|
| 540 |
+
|
| 541 |
+
self.loraplus_lr_ratio = None
|
| 542 |
+
# self.loraplus_unet_lr_ratio = None
|
| 543 |
+
# self.loraplus_text_encoder_lr_ratio = None
|
| 544 |
+
|
| 545 |
+
if modules_dim is not None:
|
| 546 |
+
logger.info("create LoRA network from weights")
|
| 547 |
+
else:
|
| 548 |
+
logger.info(f"create LoRA network. base dim (rank): {lora_dim}, alpha: {alpha}")
|
| 549 |
+
logger.info(
|
| 550 |
+
f"neuron dropout: p={self.dropout}, rank dropout: p={self.rank_dropout}, module dropout: p={self.module_dropout}"
|
| 551 |
+
)
|
| 552 |
+
# if self.conv_lora_dim is not None:
|
| 553 |
+
# logger.info(
|
| 554 |
+
# f"apply LoRA to Conv2d with kernel size (3,3). dim (rank): {self.conv_lora_dim}, alpha: {self.conv_alpha}"
|
| 555 |
+
# )
|
| 556 |
+
# if train_t5xxl:
|
| 557 |
+
# logger.info(f"train T5XXL as well")
|
| 558 |
+
|
| 559 |
+
# compile regular expression if specified
|
| 560 |
+
exclude_re_patterns = []
|
| 561 |
+
if exclude_patterns is not None:
|
| 562 |
+
for pattern in exclude_patterns:
|
| 563 |
+
try:
|
| 564 |
+
re_pattern = re.compile(pattern)
|
| 565 |
+
except re.error as e:
|
| 566 |
+
logger.error(f"Invalid exclude pattern '{pattern}': {e}")
|
| 567 |
+
continue
|
| 568 |
+
exclude_re_patterns.append(re_pattern)
|
| 569 |
+
|
| 570 |
+
include_re_patterns = []
|
| 571 |
+
if include_patterns is not None:
|
| 572 |
+
for pattern in include_patterns:
|
| 573 |
+
try:
|
| 574 |
+
re_pattern = re.compile(pattern)
|
| 575 |
+
except re.error as e:
|
| 576 |
+
logger.error(f"Invalid include pattern '{pattern}': {e}")
|
| 577 |
+
continue
|
| 578 |
+
include_re_patterns.append(re_pattern)
|
| 579 |
+
|
| 580 |
+
# create module instances
|
| 581 |
+
def create_modules(
|
| 582 |
+
is_unet: bool,
|
| 583 |
+
pfx: str,
|
| 584 |
+
root_module: torch.nn.Module,
|
| 585 |
+
target_replace_mods: Optional[List[str]] = None,
|
| 586 |
+
filter: Optional[str] = None,
|
| 587 |
+
default_dim: Optional[int] = None,
|
| 588 |
+
) -> List[LoRAModule]:
|
| 589 |
+
loras = []
|
| 590 |
+
skipped = []
|
| 591 |
+
for name, module in root_module.named_modules():
|
| 592 |
+
if target_replace_mods is None or module.__class__.__name__ in target_replace_mods:
|
| 593 |
+
if target_replace_mods is None: # dirty hack for all modules
|
| 594 |
+
module = root_module # search all modules
|
| 595 |
+
|
| 596 |
+
for child_name, child_module in module.named_modules():
|
| 597 |
+
is_linear = child_module.__class__.__name__ == "Linear"
|
| 598 |
+
is_conv2d = child_module.__class__.__name__ == "Conv2d"
|
| 599 |
+
is_conv3d = child_module.__class__.__name__ == "Conv3d"
|
| 600 |
+
is_conv2d_1x1 = is_conv2d and child_module.kernel_size == (1, 1)
|
| 601 |
+
is_conv3d_1x1 = is_conv3d and child_module.kernel_size == (1, 1, 1)
|
| 602 |
+
|
| 603 |
+
if is_linear or is_conv2d or is_conv3d:
|
| 604 |
+
original_name = (name + "." if name else "") + child_name
|
| 605 |
+
lora_name = f"{pfx}.{original_name}".replace(".", "_")
|
| 606 |
+
|
| 607 |
+
# exclude/include filter
|
| 608 |
+
excluded = False
|
| 609 |
+
for pattern in exclude_re_patterns:
|
| 610 |
+
if pattern.fullmatch(original_name):
|
| 611 |
+
excluded = True
|
| 612 |
+
break
|
| 613 |
+
included = False
|
| 614 |
+
for pattern in include_re_patterns:
|
| 615 |
+
if pattern.fullmatch(original_name):
|
| 616 |
+
included = True
|
| 617 |
+
break
|
| 618 |
+
if excluded and not included:
|
| 619 |
+
if verbose:
|
| 620 |
+
logger.info(f"exclude: {original_name}")
|
| 621 |
+
continue
|
| 622 |
+
|
| 623 |
+
# filter by name (not used in the current implementation)
|
| 624 |
+
if filter is not None and filter not in lora_name:
|
| 625 |
+
continue
|
| 626 |
+
|
| 627 |
+
dim = None
|
| 628 |
+
alpha = None
|
| 629 |
+
|
| 630 |
+
if modules_dim is not None:
|
| 631 |
+
# モジュール指定あり
|
| 632 |
+
if lora_name in modules_dim:
|
| 633 |
+
dim = modules_dim[lora_name]
|
| 634 |
+
alpha = modules_alpha[lora_name]
|
| 635 |
+
else:
|
| 636 |
+
# 通常、すべて対象とする
|
| 637 |
+
if is_linear or is_conv2d_1x1 or is_conv3d_1x1:
|
| 638 |
+
dim = default_dim if default_dim is not None else self.lora_dim
|
| 639 |
+
alpha = self.alpha
|
| 640 |
+
elif self.conv_lora_dim is not None:
|
| 641 |
+
dim = self.conv_lora_dim
|
| 642 |
+
alpha = self.conv_alpha
|
| 643 |
+
|
| 644 |
+
if dim is None or dim == 0:
|
| 645 |
+
# skipした情報を出力
|
| 646 |
+
if is_linear or is_conv2d_1x1 or is_conv3d_1x1 or (self.conv_lora_dim is not None):
|
| 647 |
+
skipped.append(lora_name)
|
| 648 |
+
continue
|
| 649 |
+
|
| 650 |
+
lora = module_class(
|
| 651 |
+
lora_name,
|
| 652 |
+
child_module,
|
| 653 |
+
self.multiplier,
|
| 654 |
+
dim,
|
| 655 |
+
alpha,
|
| 656 |
+
dropout=dropout,
|
| 657 |
+
rank_dropout=rank_dropout,
|
| 658 |
+
module_dropout=module_dropout,
|
| 659 |
+
**self.module_kwargs,
|
| 660 |
+
)
|
| 661 |
+
loras.append(lora)
|
| 662 |
+
|
| 663 |
+
if target_replace_mods is None:
|
| 664 |
+
break # all modules are searched
|
| 665 |
+
return loras, skipped
|
| 666 |
+
|
| 667 |
+
# # create LoRA for text encoder
|
| 668 |
+
# # it is redundant to create LoRA modules even if they are not used
|
| 669 |
+
|
| 670 |
+
self.text_encoder_loras: List[Union[LoRAModule, LoRAInfModule]] = []
|
| 671 |
+
# skipped_te = []
|
| 672 |
+
# for i, text_encoder in enumerate(text_encoders):
|
| 673 |
+
# index = i
|
| 674 |
+
# if not train_t5xxl and index > 0: # 0: CLIP, 1: T5XXL, so we skip T5XXL if train_t5xxl is False
|
| 675 |
+
# break
|
| 676 |
+
# logger.info(f"create LoRA for Text Encoder {index+1}:")
|
| 677 |
+
# text_encoder_loras, skipped = create_modules(False, index, text_encoder, LoRANetwork.TEXT_ENCODER_TARGET_REPLACE_MODULE)
|
| 678 |
+
# logger.info(f"create LoRA for Text Encoder {index+1}: {len(text_encoder_loras)} modules.")
|
| 679 |
+
# self.text_encoder_loras.extend(text_encoder_loras)
|
| 680 |
+
# skipped_te += skipped
|
| 681 |
+
|
| 682 |
+
# create LoRA for U-Net
|
| 683 |
+
self.unet_loras: List[Union[LoRAModule, LoRAInfModule]]
|
| 684 |
+
self.unet_loras, skipped_un = create_modules(True, prefix, unet, target_replace_modules)
|
| 685 |
+
|
| 686 |
+
logger.info(f"create LoRA for U-Net/DiT: {len(self.unet_loras)} modules.")
|
| 687 |
+
if verbose:
|
| 688 |
+
for lora in self.unet_loras:
|
| 689 |
+
logger.info(f"\t{lora.lora_name:50} {lora.lora_dim}, {lora.alpha}")
|
| 690 |
+
|
| 691 |
+
skipped = skipped_un
|
| 692 |
+
if verbose and len(skipped) > 0:
|
| 693 |
+
logger.warning(
|
| 694 |
+
f"because dim (rank) is 0, {len(skipped)} LoRA modules are skipped / dim (rank)が0の為、次の{len(skipped)}個のLoRAモジュールはスキップされます:"
|
| 695 |
+
)
|
| 696 |
+
for name in skipped:
|
| 697 |
+
logger.info(f"\t{name}")
|
| 698 |
+
|
| 699 |
+
# assertion
|
| 700 |
+
names = set()
|
| 701 |
+
for lora in self.text_encoder_loras + self.unet_loras:
|
| 702 |
+
assert lora.lora_name not in names, f"duplicated lora name: {lora.lora_name}"
|
| 703 |
+
names.add(lora.lora_name)
|
| 704 |
+
|
| 705 |
+
def prepare_network(self, args):
|
| 706 |
+
"""
|
| 707 |
+
called after the network is created
|
| 708 |
+
"""
|
| 709 |
+
pass
|
| 710 |
+
|
| 711 |
+
def set_multiplier(self, multiplier):
|
| 712 |
+
self.multiplier = multiplier
|
| 713 |
+
for lora in self.text_encoder_loras + self.unet_loras:
|
| 714 |
+
lora.multiplier = self.multiplier
|
| 715 |
+
|
| 716 |
+
def set_enabled(self, is_enabled):
|
| 717 |
+
for lora in self.text_encoder_loras + self.unet_loras:
|
| 718 |
+
lora.enabled = is_enabled
|
| 719 |
+
|
| 720 |
+
def load_weights(self, file):
|
| 721 |
+
if os.path.splitext(file)[1] == ".safetensors":
|
| 722 |
+
from safetensors.torch import load_file
|
| 723 |
+
|
| 724 |
+
weights_sd = load_file(file)
|
| 725 |
+
else:
|
| 726 |
+
weights_sd = torch.load(file, map_location="cpu")
|
| 727 |
+
|
| 728 |
+
info = self.load_state_dict(weights_sd, False)
|
| 729 |
+
return info
|
| 730 |
+
|
| 731 |
+
def apply_to(
|
| 732 |
+
self,
|
| 733 |
+
text_encoders: Optional[nn.Module],
|
| 734 |
+
unet: Optional[nn.Module],
|
| 735 |
+
apply_text_encoder: bool = True,
|
| 736 |
+
apply_unet: bool = True,
|
| 737 |
+
):
|
| 738 |
+
if apply_text_encoder:
|
| 739 |
+
logger.info(f"enable LoRA for text encoder: {len(self.text_encoder_loras)} modules")
|
| 740 |
+
else:
|
| 741 |
+
self.text_encoder_loras = []
|
| 742 |
+
|
| 743 |
+
if apply_unet:
|
| 744 |
+
logger.info(f"enable LoRA for U-Net: {len(self.unet_loras)} modules")
|
| 745 |
+
else:
|
| 746 |
+
self.unet_loras = []
|
| 747 |
+
|
| 748 |
+
if len(self.text_encoder_loras) == 0 and len(self.unet_loras) == 0:
|
| 749 |
+
logger.error(
|
| 750 |
+
"No LoRA modules. Please check `--network_module` and `--network_args`"
|
| 751 |
+
" / LoRAモジュールがありません。`--network_module`と`--network_args`を確認してください"
|
| 752 |
+
)
|
| 753 |
+
raise RuntimeError("No LoRA modules found")
|
| 754 |
+
|
| 755 |
+
for lora in self.text_encoder_loras + self.unet_loras:
|
| 756 |
+
lora.apply_to()
|
| 757 |
+
self.add_module(lora.lora_name, lora)
|
| 758 |
+
|
| 759 |
+
# マージできるかどうかを返す
|
| 760 |
+
def is_mergeable(self):
|
| 761 |
+
return True
|
| 762 |
+
|
| 763 |
+
# TODO refactor to common function with apply_to
|
| 764 |
+
def merge_to(self, text_encoders, unet, weights_sd, dtype=None, device=None, non_blocking=False):
|
| 765 |
+
from concurrent.futures import ThreadPoolExecutor
|
| 766 |
+
|
| 767 |
+
with ThreadPoolExecutor(max_workers=2) as executor: # 2 workers is enough
|
| 768 |
+
futures = []
|
| 769 |
+
for lora in self.text_encoder_loras + self.unet_loras:
|
| 770 |
+
sd_for_lora = {}
|
| 771 |
+
for key in weights_sd.keys():
|
| 772 |
+
if key.startswith(lora.lora_name):
|
| 773 |
+
sd_for_lora[key[len(lora.lora_name) + 1 :]] = weights_sd[key]
|
| 774 |
+
if len(sd_for_lora) == 0:
|
| 775 |
+
logger.info(f"no weight for {lora.lora_name}")
|
| 776 |
+
continue
|
| 777 |
+
|
| 778 |
+
# lora.merge_to(sd_for_lora, dtype, device)
|
| 779 |
+
futures.append(executor.submit(lora.merge_to, sd_for_lora, dtype, device, non_blocking))
|
| 780 |
+
|
| 781 |
+
for future in futures:
|
| 782 |
+
future.result()
|
| 783 |
+
|
| 784 |
+
logger.info("weights are merged")
|
| 785 |
+
|
| 786 |
+
def set_loraplus_lr_ratio(self, loraplus_lr_ratio): # , loraplus_unet_lr_ratio, loraplus_text_encoder_lr_ratio):
|
| 787 |
+
self.loraplus_lr_ratio = loraplus_lr_ratio
|
| 788 |
+
|
| 789 |
+
logger.info(f"LoRA+ UNet LR Ratio: {self.loraplus_lr_ratio}")
|
| 790 |
+
# logger.info(f"LoRA+ Text Encoder LR Ratio: {self.loraplus_text_encoder_lr_ratio or self.loraplus_lr_ratio}")
|
| 791 |
+
|
| 792 |
+
def prepare_optimizer_params(self, unet_lr: float = 1e-4, **kwargs):
|
| 793 |
+
self.requires_grad_(True)
|
| 794 |
+
|
| 795 |
+
all_params = []
|
| 796 |
+
lr_descriptions = []
|
| 797 |
+
|
| 798 |
+
def assemble_params(loras, lr, loraplus_ratio):
|
| 799 |
+
param_groups = {"lora": {}, "plus": {}}
|
| 800 |
+
for lora in loras:
|
| 801 |
+
for name, param in lora.named_parameters():
|
| 802 |
+
if loraplus_ratio is not None and "lora_up" in name:
|
| 803 |
+
param_groups["plus"][f"{lora.lora_name}.{name}"] = param
|
| 804 |
+
else:
|
| 805 |
+
param_groups["lora"][f"{lora.lora_name}.{name}"] = param
|
| 806 |
+
|
| 807 |
+
if loraplus_ratio is not None and len(param_groups["plus"]) == 0:
|
| 808 |
+
logger.warning("LoRA+ is not effective for this network type (no 'lora_up' parameters found)")
|
| 809 |
+
|
| 810 |
+
params = []
|
| 811 |
+
descriptions = []
|
| 812 |
+
for key in param_groups.keys():
|
| 813 |
+
param_data = {"params": param_groups[key].values()}
|
| 814 |
+
|
| 815 |
+
if len(param_data["params"]) == 0:
|
| 816 |
+
continue
|
| 817 |
+
|
| 818 |
+
if lr is not None:
|
| 819 |
+
if key == "plus":
|
| 820 |
+
param_data["lr"] = lr * loraplus_ratio
|
| 821 |
+
else:
|
| 822 |
+
param_data["lr"] = lr
|
| 823 |
+
|
| 824 |
+
if param_data.get("lr", None) == 0 or param_data.get("lr", None) is None:
|
| 825 |
+
logger.info("NO LR skipping!")
|
| 826 |
+
continue
|
| 827 |
+
|
| 828 |
+
params.append(param_data)
|
| 829 |
+
descriptions.append("plus" if key == "plus" else "")
|
| 830 |
+
|
| 831 |
+
return params, descriptions
|
| 832 |
+
|
| 833 |
+
if self.unet_loras:
|
| 834 |
+
params, descriptions = assemble_params(self.unet_loras, unet_lr, self.loraplus_lr_ratio)
|
| 835 |
+
all_params.extend(params)
|
| 836 |
+
lr_descriptions.extend(["unet" + (" " + d if d else "") for d in descriptions])
|
| 837 |
+
|
| 838 |
+
return all_params, lr_descriptions
|
| 839 |
+
|
| 840 |
+
def enable_gradient_checkpointing(self):
|
| 841 |
+
# not supported
|
| 842 |
+
pass
|
| 843 |
+
|
| 844 |
+
def prepare_grad_etc(self, unet):
|
| 845 |
+
self.requires_grad_(True)
|
| 846 |
+
|
| 847 |
+
def on_epoch_start(self, unet):
|
| 848 |
+
self.train()
|
| 849 |
+
|
| 850 |
+
def on_step_start(self):
|
| 851 |
+
pass
|
| 852 |
+
|
| 853 |
+
def get_trainable_params(self):
|
| 854 |
+
return self.parameters()
|
| 855 |
+
|
| 856 |
+
def save_weights(self, file, dtype, metadata):
|
| 857 |
+
if metadata is not None and len(metadata) == 0:
|
| 858 |
+
metadata = None
|
| 859 |
+
|
| 860 |
+
state_dict = self.state_dict()
|
| 861 |
+
|
| 862 |
+
if dtype is not None:
|
| 863 |
+
for key in list(state_dict.keys()):
|
| 864 |
+
v = state_dict[key]
|
| 865 |
+
v = v.detach().clone().to("cpu").to(dtype)
|
| 866 |
+
state_dict[key] = v
|
| 867 |
+
|
| 868 |
+
if os.path.splitext(file)[1] == ".safetensors":
|
| 869 |
+
from safetensors.torch import save_file
|
| 870 |
+
from musubi_tuner.utils import model_utils
|
| 871 |
+
|
| 872 |
+
# Precalculate model hashes to save time on indexing
|
| 873 |
+
if metadata is None:
|
| 874 |
+
metadata = {}
|
| 875 |
+
model_hash, legacy_hash = model_utils.precalculate_safetensors_hashes(state_dict, metadata)
|
| 876 |
+
metadata["sshs_model_hash"] = model_hash
|
| 877 |
+
metadata["sshs_legacy_hash"] = legacy_hash
|
| 878 |
+
|
| 879 |
+
save_file(state_dict, file, metadata)
|
| 880 |
+
else:
|
| 881 |
+
torch.save(state_dict, file)
|
| 882 |
+
|
| 883 |
+
def backup_weights(self):
|
| 884 |
+
# 重みのバックアップを行う
|
| 885 |
+
loras: List[LoRAInfModule] = self.text_encoder_loras + self.unet_loras
|
| 886 |
+
for lora in loras:
|
| 887 |
+
org_module = lora.org_module_ref[0]
|
| 888 |
+
if not hasattr(org_module, "_lora_org_weight"):
|
| 889 |
+
sd = org_module.state_dict()
|
| 890 |
+
org_module._lora_org_weight = sd["weight"].detach().clone()
|
| 891 |
+
org_module._lora_restored = True
|
| 892 |
+
|
| 893 |
+
def restore_weights(self):
|
| 894 |
+
# 重みのリストアを行う
|
| 895 |
+
loras: List[LoRAInfModule] = self.text_encoder_loras + self.unet_loras
|
| 896 |
+
for lora in loras:
|
| 897 |
+
org_module = lora.org_module_ref[0]
|
| 898 |
+
if not org_module._lora_restored:
|
| 899 |
+
sd = org_module.state_dict()
|
| 900 |
+
sd["weight"] = org_module._lora_org_weight
|
| 901 |
+
org_module.load_state_dict(sd)
|
| 902 |
+
org_module._lora_restored = True
|
| 903 |
+
|
| 904 |
+
def pre_calculation(self):
|
| 905 |
+
# 事前計算を行う
|
| 906 |
+
loras: List[LoRAInfModule] = self.text_encoder_loras + self.unet_loras
|
| 907 |
+
for lora in loras:
|
| 908 |
+
org_module = lora.org_module_ref[0]
|
| 909 |
+
sd = org_module.state_dict()
|
| 910 |
+
|
| 911 |
+
org_weight = sd["weight"]
|
| 912 |
+
lora_weight = lora.get_weight().to(org_weight.device, dtype=org_weight.dtype)
|
| 913 |
+
sd["weight"] = org_weight + lora_weight
|
| 914 |
+
assert sd["weight"].shape == org_weight.shape
|
| 915 |
+
org_module.load_state_dict(sd)
|
| 916 |
+
|
| 917 |
+
org_module._lora_restored = False
|
| 918 |
+
lora.enabled = False
|
| 919 |
+
|
| 920 |
+
def apply_max_norm_regularization(self, max_norm_value, device):
|
| 921 |
+
downkeys = []
|
| 922 |
+
upkeys = []
|
| 923 |
+
alphakeys = []
|
| 924 |
+
norms = []
|
| 925 |
+
keys_scaled = 0
|
| 926 |
+
|
| 927 |
+
state_dict = self.state_dict()
|
| 928 |
+
|
| 929 |
+
# guard: only supported for LoRA (lora_down/lora_up parameterization)
|
| 930 |
+
if not any("lora_down" in k and "weight" in k for k in state_dict.keys()):
|
| 931 |
+
logger.warning("max_norm_regularization is only supported for LoRA")
|
| 932 |
+
return 0, 0.0, 0.0
|
| 933 |
+
|
| 934 |
+
for key in state_dict.keys():
|
| 935 |
+
if "lora_down" in key and "weight" in key:
|
| 936 |
+
downkeys.append(key)
|
| 937 |
+
upkeys.append(key.replace("lora_down", "lora_up"))
|
| 938 |
+
alphakeys.append(key.replace("lora_down.weight", "alpha"))
|
| 939 |
+
|
| 940 |
+
for i in range(len(downkeys)):
|
| 941 |
+
down = state_dict[downkeys[i]].to(device)
|
| 942 |
+
up = state_dict[upkeys[i]].to(device)
|
| 943 |
+
alpha = state_dict[alphakeys[i]].to(device)
|
| 944 |
+
dim = down.shape[0]
|
| 945 |
+
scale = alpha / dim
|
| 946 |
+
|
| 947 |
+
if up.shape[2:] == (1, 1) and down.shape[2:] == (1, 1):
|
| 948 |
+
updown = (up.squeeze(2).squeeze(2) @ down.squeeze(2).squeeze(2)).unsqueeze(2).unsqueeze(3)
|
| 949 |
+
elif up.shape[2:] == (3, 3) or down.shape[2:] == (3, 3):
|
| 950 |
+
updown = torch.nn.functional.conv2d(down.permute(1, 0, 2, 3), up).permute(1, 0, 2, 3)
|
| 951 |
+
else:
|
| 952 |
+
updown = up @ down
|
| 953 |
+
|
| 954 |
+
updown *= scale
|
| 955 |
+
|
| 956 |
+
norm = updown.norm().clamp(min=max_norm_value / 2)
|
| 957 |
+
desired = torch.clamp(norm, max=max_norm_value)
|
| 958 |
+
ratio = desired.cpu() / norm.cpu()
|
| 959 |
+
sqrt_ratio = ratio**0.5
|
| 960 |
+
if ratio != 1:
|
| 961 |
+
keys_scaled += 1
|
| 962 |
+
state_dict[upkeys[i]] *= sqrt_ratio
|
| 963 |
+
state_dict[downkeys[i]] *= sqrt_ratio
|
| 964 |
+
scalednorm = updown.norm() * ratio
|
| 965 |
+
norms.append(scalednorm.item())
|
| 966 |
+
|
| 967 |
+
return keys_scaled, sum(norms) / len(norms), max(norms)
|
| 968 |
+
|
| 969 |
+
|
| 970 |
+
def create_arch_network_from_weights(
|
| 971 |
+
multiplier: float,
|
| 972 |
+
weights_sd: Dict[str, torch.Tensor],
|
| 973 |
+
text_encoders: Optional[List[nn.Module]] = None,
|
| 974 |
+
unet: Optional[nn.Module] = None,
|
| 975 |
+
for_inference: bool = False,
|
| 976 |
+
**kwargs,
|
| 977 |
+
) -> LoRANetwork:
|
| 978 |
+
return create_network_from_weights(
|
| 979 |
+
HUNYUAN_TARGET_REPLACE_MODULES, multiplier, weights_sd, text_encoders, unet, for_inference, **kwargs
|
| 980 |
+
)
|
| 981 |
+
|
| 982 |
+
|
| 983 |
+
# Create network from weights for inference, weights are not loaded here (because can be merged)
|
| 984 |
+
def create_network_from_weights(
|
| 985 |
+
target_replace_modules: List[str],
|
| 986 |
+
multiplier: float,
|
| 987 |
+
weights_sd: Dict[str, torch.Tensor],
|
| 988 |
+
text_encoders: Optional[List[nn.Module]] = None,
|
| 989 |
+
unet: Optional[nn.Module] = None,
|
| 990 |
+
for_inference: bool = False,
|
| 991 |
+
module_class: Optional[Type[object]] = None,
|
| 992 |
+
module_kwargs: Optional[Dict[str, Any]] = None,
|
| 993 |
+
**kwargs,
|
| 994 |
+
) -> LoRANetwork:
|
| 995 |
+
# get dim/alpha mapping
|
| 996 |
+
modules_dim = {}
|
| 997 |
+
modules_alpha = {}
|
| 998 |
+
for key, value in weights_sd.items():
|
| 999 |
+
if "." not in key:
|
| 1000 |
+
continue
|
| 1001 |
+
|
| 1002 |
+
lora_name = key.split(".")[0]
|
| 1003 |
+
if "alpha" in key:
|
| 1004 |
+
modules_alpha[lora_name] = value
|
| 1005 |
+
elif "lora_down" in key:
|
| 1006 |
+
dim = value.shape[0]
|
| 1007 |
+
modules_dim[lora_name] = dim
|
| 1008 |
+
# logger.info(lora_name, value.size(), dim)
|
| 1009 |
+
|
| 1010 |
+
if module_class is None:
|
| 1011 |
+
module_class = LoRAInfModule if for_inference else LoRAModule
|
| 1012 |
+
|
| 1013 |
+
network = LoRANetwork(
|
| 1014 |
+
target_replace_modules,
|
| 1015 |
+
"lora_unet",
|
| 1016 |
+
text_encoders,
|
| 1017 |
+
unet,
|
| 1018 |
+
multiplier=multiplier,
|
| 1019 |
+
modules_dim=modules_dim,
|
| 1020 |
+
modules_alpha=modules_alpha,
|
| 1021 |
+
module_class=module_class,
|
| 1022 |
+
module_kwargs=module_kwargs,
|
| 1023 |
+
)
|
| 1024 |
+
return network
|
third_party/musubi_tuner/networks/lora_minimax_h3.py
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import ast
|
| 2 |
+
import logging
|
| 3 |
+
from typing import Dict, List, Optional
|
| 4 |
+
|
| 5 |
+
import torch
|
| 6 |
+
import torch.nn as nn
|
| 7 |
+
|
| 8 |
+
from musubi_tuner import convert_lora
|
| 9 |
+
import musubi_tuner.networks.lora as lora
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
logger = logging.getLogger(__name__)
|
| 13 |
+
|
| 14 |
+
MINIMAX_H3_TARGET_REPLACE_MODULES = ["DiTBlock"]
|
| 15 |
+
MINIMAX_H3_DEFAULT_TARGET_PATTERN = r"blocks\.\d+\.(?:attn\.(?:qkv_proj|out_proj)|mlp\.(?:fc1|fc2))"
|
| 16 |
+
_DEFAULT_EXCLUDE_PATTERN = rf"(?!{MINIMAX_H3_DEFAULT_TARGET_PATTERN}$).*"
|
| 17 |
+
# Diffusers-format H3 LoRA keys, the format of the third-party training adapters and of
|
| 18 |
+
# ai-toolkit / diffusion-pipe LoRAs: `diffusion_model.blocks.N.attn.qkv_proj.lora_A.weight`
|
| 19 |
+
_DIFFUSERS_KEY_PREFIXES = ("diffusion_model.", "transformer.")
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def convert_lora_state_dict(weights_sd: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]:
|
| 23 |
+
"""Return ``weights_sd`` in the native ``lora_unet_*`` key format.
|
| 24 |
+
|
| 25 |
+
Native state dicts pass through untouched. Diffusers-format ones (``diffusion_model.`` or
|
| 26 |
+
``transformer.`` prefix, ``lora_A``/``lora_B``, no alpha) are renamed to the Musubi keys
|
| 27 |
+
with ``alpha = rank`` filled in, the same treatment ``hv_train_network`` gives HunyuanVideo
|
| 28 |
+
LoRAs. Every H3 LoRA entry point (``--base_weights``, ``--lora_weight`` on each of its
|
| 29 |
+
routes) goes through here so the third-party adapters load without a manual conversion.
|
| 30 |
+
"""
|
| 31 |
+
if not weights_sd:
|
| 32 |
+
return weights_sd
|
| 33 |
+
first_key = next(iter(weights_sd))
|
| 34 |
+
if first_key.startswith("lora_"):
|
| 35 |
+
return weights_sd
|
| 36 |
+
if first_key.startswith(_DIFFUSERS_KEY_PREFIXES):
|
| 37 |
+
logger.info("Converting MiniMax-H3 LoRA weights from the Diffusers key format to the native format")
|
| 38 |
+
return convert_lora.convert_from_diffusers("lora_unet_", weights_sd)
|
| 39 |
+
return weights_sd
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def _pattern_list(value) -> list[str]:
|
| 43 |
+
if value is None:
|
| 44 |
+
return []
|
| 45 |
+
if isinstance(value, str):
|
| 46 |
+
value = ast.literal_eval(value)
|
| 47 |
+
return list(value)
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def create_arch_network(
|
| 51 |
+
multiplier: float,
|
| 52 |
+
network_dim: Optional[int],
|
| 53 |
+
network_alpha: Optional[float],
|
| 54 |
+
vae: nn.Module,
|
| 55 |
+
text_encoders: List[nn.Module],
|
| 56 |
+
unet: nn.Module,
|
| 57 |
+
neuron_dropout: Optional[float] = None,
|
| 58 |
+
**kwargs,
|
| 59 |
+
):
|
| 60 |
+
exclude_patterns = _pattern_list(kwargs.get("exclude_patterns"))
|
| 61 |
+
exclude_patterns.append(_DEFAULT_EXCLUDE_PATTERN)
|
| 62 |
+
kwargs["exclude_patterns"] = exclude_patterns
|
| 63 |
+
network = lora.create_network(
|
| 64 |
+
MINIMAX_H3_TARGET_REPLACE_MODULES,
|
| 65 |
+
"lora_unet",
|
| 66 |
+
multiplier,
|
| 67 |
+
network_dim,
|
| 68 |
+
network_alpha,
|
| 69 |
+
vae,
|
| 70 |
+
text_encoders,
|
| 71 |
+
unet,
|
| 72 |
+
neuron_dropout=neuron_dropout,
|
| 73 |
+
**kwargs,
|
| 74 |
+
)
|
| 75 |
+
if not network.unet_loras:
|
| 76 |
+
raise RuntimeError("MiniMax-H3 LoRA found zero target modules; check the target include/exclude patterns")
|
| 77 |
+
return network
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def create_arch_network_from_weights(
|
| 81 |
+
multiplier: float,
|
| 82 |
+
weights_sd: Dict[str, torch.Tensor],
|
| 83 |
+
text_encoders: Optional[List[nn.Module]] = None,
|
| 84 |
+
unet: Optional[nn.Module] = None,
|
| 85 |
+
for_inference: bool = False,
|
| 86 |
+
**kwargs,
|
| 87 |
+
) -> lora.LoRANetwork:
|
| 88 |
+
# Search every Linear of the model, not just the DiTBlock targets of the training default:
|
| 89 |
+
# the modules are created from the weights, so a saved LoRA decides its own coverage, and
|
| 90 |
+
# third-party LoRAs (ai-toolkit and the training adapters) also carry token_refiner modules.
|
| 91 |
+
return lora.create_network_from_weights(
|
| 92 |
+
None,
|
| 93 |
+
multiplier,
|
| 94 |
+
weights_sd,
|
| 95 |
+
text_encoders,
|
| 96 |
+
unet,
|
| 97 |
+
for_inference,
|
| 98 |
+
**kwargs,
|
| 99 |
+
)
|
third_party/musubi_tuner/utils/__init__.py
ADDED
|
File without changes
|
third_party/musubi_tuner/utils/cv2_compat.py
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
OpenCV (``cv2``) optional-dependency bridge.
|
| 3 |
+
|
| 4 |
+
Importing this module has one of two effects:
|
| 5 |
+
|
| 6 |
+
1. If ``opencv-python`` is installed, nothing changes -- the real ``cv2``
|
| 7 |
+
module is used as-is, and :data:`HAS_OPENCV` is set to ``True``.
|
| 8 |
+
|
| 9 |
+
2. If it is not installed, a lightweight stub implemented in
|
| 10 |
+
:mod:`musubi_tuner.utils._cv2_stub` is registered as ``cv2`` in
|
| 11 |
+
:data:`sys.modules`, so any subsequent ``import cv2`` call transparently
|
| 12 |
+
picks it up. :data:`HAS_OPENCV` is set to ``False``.
|
| 13 |
+
|
| 14 |
+
Import this module BEFORE ``import cv2`` in any file that wants to tolerate
|
| 15 |
+
a missing OpenCV installation::
|
| 16 |
+
|
| 17 |
+
from musubi_tuner.utils import cv2_compat # noqa: F401 - must come before `import cv2`
|
| 18 |
+
import cv2
|
| 19 |
+
|
| 20 |
+
Tools that genuinely require OpenCV should bail out early with a clear
|
| 21 |
+
message when :data:`HAS_OPENCV` is ``False``.
|
| 22 |
+
"""
|
| 23 |
+
|
| 24 |
+
from __future__ import annotations
|
| 25 |
+
|
| 26 |
+
import logging
|
| 27 |
+
import sys
|
| 28 |
+
|
| 29 |
+
logger = logging.getLogger(__name__)
|
| 30 |
+
|
| 31 |
+
try:
|
| 32 |
+
import cv2 as _real_cv2 # noqa: F401 # real OpenCV is importable
|
| 33 |
+
|
| 34 |
+
HAS_OPENCV = True
|
| 35 |
+
except ImportError:
|
| 36 |
+
HAS_OPENCV = False
|
| 37 |
+
if "cv2" not in sys.modules:
|
| 38 |
+
from musubi_tuner.utils import _cv2_stub
|
| 39 |
+
|
| 40 |
+
sys.modules["cv2"] = _cv2_stub
|
| 41 |
+
logger.info(
|
| 42 |
+
"opencv-python is not installed; using the built-in cv2 stub "
|
| 43 |
+
"(Pillow/NumPy fallback). Some tools that rely on the real "
|
| 44 |
+
"OpenCV will not work."
|
| 45 |
+
)
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def require_opencv(tool_name: str = "this tool") -> None:
|
| 49 |
+
"""Abort execution with a clear message when real OpenCV is missing."""
|
| 50 |
+
if HAS_OPENCV:
|
| 51 |
+
return
|
| 52 |
+
raise SystemExit(
|
| 53 |
+
f"{tool_name} requires opencv-python, but it is not installed.\n"
|
| 54 |
+
f"{tool_name} は opencv-python が必要ですが、インストールされていません。\n"
|
| 55 |
+
"Install it with: pip install opencv-python"
|
| 56 |
+
)
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
__all__ = ["HAS_OPENCV", "require_opencv"]
|
third_party/musubi_tuner/utils/device_utils.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Optional, Union
|
| 2 |
+
import torch
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
def clean_memory_on_device(device: Optional[Union[str, torch.device]]):
|
| 6 |
+
if device is None:
|
| 7 |
+
return
|
| 8 |
+
if isinstance(device, str):
|
| 9 |
+
device = torch.device(device)
|
| 10 |
+
if device.type == "cuda":
|
| 11 |
+
torch.cuda.empty_cache()
|
| 12 |
+
elif device.type == "cpu":
|
| 13 |
+
pass
|
| 14 |
+
elif device.type == "mps": # not tested
|
| 15 |
+
torch.mps.empty_cache()
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def synchronize_device(device: Optional[Union[str, torch.device]]):
|
| 19 |
+
if device is None:
|
| 20 |
+
return
|
| 21 |
+
if isinstance(device, str):
|
| 22 |
+
device = torch.device(device)
|
| 23 |
+
if device.type == "cuda":
|
| 24 |
+
torch.cuda.synchronize()
|
| 25 |
+
elif device.type == "xpu":
|
| 26 |
+
torch.xpu.synchronize()
|
| 27 |
+
elif device.type == "mps":
|
| 28 |
+
torch.mps.synchronize()
|
third_party/musubi_tuner/utils/lora_utils.py
ADDED
|
@@ -0,0 +1,368 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import re
|
| 3 |
+
from types import ModuleType
|
| 4 |
+
from typing import Callable, Dict, List, Optional, Union
|
| 5 |
+
import torch
|
| 6 |
+
|
| 7 |
+
import logging
|
| 8 |
+
|
| 9 |
+
from safetensors.torch import load_file
|
| 10 |
+
from tqdm import tqdm
|
| 11 |
+
|
| 12 |
+
from musubi_tuner.utils.device_utils import synchronize_device
|
| 13 |
+
|
| 14 |
+
logger = logging.getLogger(__name__)
|
| 15 |
+
logging.basicConfig(level=logging.INFO)
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
from musubi_tuner.modules.fp8_optimization_utils import load_safetensors_with_fp8_optimization
|
| 19 |
+
from musubi_tuner.utils.safetensors_utils import (
|
| 20 |
+
MemoryEfficientSafeOpen,
|
| 21 |
+
TensorWeightAdapter,
|
| 22 |
+
WeightTransformHooks,
|
| 23 |
+
get_split_weight_filenames,
|
| 24 |
+
)
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def detect_network_type(lora_sd: Dict[str, torch.Tensor]) -> str:
|
| 28 |
+
"""Detect network type (lora, loha, lokr) from state dict keys."""
|
| 29 |
+
for key in lora_sd:
|
| 30 |
+
if "lora_down" in key:
|
| 31 |
+
return "lora"
|
| 32 |
+
if "hada_w1_a" in key:
|
| 33 |
+
return "loha"
|
| 34 |
+
if "lokr_w1" in key:
|
| 35 |
+
return "lokr"
|
| 36 |
+
return "lora" # default
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def filter_lora_state_dict(
|
| 40 |
+
weights_sd: Dict[str, torch.Tensor],
|
| 41 |
+
include_pattern: Optional[str] = None,
|
| 42 |
+
exclude_pattern: Optional[str] = None,
|
| 43 |
+
) -> Dict[str, torch.Tensor]:
|
| 44 |
+
# apply include/exclude patterns
|
| 45 |
+
original_key_count = len(weights_sd.keys())
|
| 46 |
+
if include_pattern is not None:
|
| 47 |
+
regex_include = re.compile(include_pattern)
|
| 48 |
+
weights_sd = {k: v for k, v in weights_sd.items() if regex_include.search(k)}
|
| 49 |
+
logger.info(f"Filtered keys with include pattern {include_pattern}: {original_key_count} -> {len(weights_sd.keys())}")
|
| 50 |
+
|
| 51 |
+
if exclude_pattern is not None:
|
| 52 |
+
original_key_count_ex = len(weights_sd.keys())
|
| 53 |
+
regex_exclude = re.compile(exclude_pattern)
|
| 54 |
+
weights_sd = {k: v for k, v in weights_sd.items() if not regex_exclude.search(k)}
|
| 55 |
+
logger.info(f"Filtered keys with exclude pattern {exclude_pattern}: {original_key_count_ex} -> {len(weights_sd.keys())}")
|
| 56 |
+
|
| 57 |
+
if len(weights_sd) != original_key_count:
|
| 58 |
+
remaining_keys = list(set([k.split(".", 1)[0] for k in weights_sd.keys()]))
|
| 59 |
+
remaining_keys.sort()
|
| 60 |
+
logger.info(f"Remaining LoRA modules after filtering: {remaining_keys}")
|
| 61 |
+
if len(weights_sd) == 0:
|
| 62 |
+
logger.warning("No keys left after filtering.")
|
| 63 |
+
|
| 64 |
+
return weights_sd
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def attach_lora_weights(
|
| 68 |
+
lora_module: ModuleType,
|
| 69 |
+
model: torch.nn.Module,
|
| 70 |
+
lora_weights: Optional[List[str]],
|
| 71 |
+
lora_multipliers: Optional[List[float]],
|
| 72 |
+
include_patterns: Optional[List[str]],
|
| 73 |
+
exclude_patterns: Optional[List[str]],
|
| 74 |
+
device: torch.device,
|
| 75 |
+
converter: Optional[Callable[[Dict[str, torch.Tensor]], Dict[str, torch.Tensor]]] = None,
|
| 76 |
+
) -> List[torch.nn.Module]:
|
| 77 |
+
"""Attach LoRAs to a model as runtime additive branches instead of merging them (inference).
|
| 78 |
+
|
| 79 |
+
The base weights are never modified: each LoRA keeps its own multiplier and precision for
|
| 80 |
+
the lifetime of the returned networks. This is the route for bases that cannot be merged
|
| 81 |
+
into (pre-quantized INT8/FP8 tensors) and for small-magnitude adapters whose per-element
|
| 82 |
+
deltas a merge would round away on the base storage grid. Multiple LoRAs stack, each
|
| 83 |
+
wrapping the previous forward. The caller keeps the returned networks alive while sampling.
|
| 84 |
+
|
| 85 |
+
Args:
|
| 86 |
+
lora_module: architecture LoRA module, e.g. lora_wan (provides create_arch_network_from_weights)
|
| 87 |
+
model: DiT model
|
| 88 |
+
lora_weights: paths to LoRA weights
|
| 89 |
+
lora_multipliers: multipliers for LoRA weights, aligned with lora_weights
|
| 90 |
+
include_patterns: regex patterns to include LoRA modules, aligned with lora_weights
|
| 91 |
+
exclude_patterns: regex patterns to exclude LoRA modules, aligned with lora_weights
|
| 92 |
+
device: device the attached networks run on
|
| 93 |
+
converter: optional state-dict key converter applied before filtering (same role as
|
| 94 |
+
the ``converter`` of ``wan_generate_video.merge_lora_weights``)
|
| 95 |
+
"""
|
| 96 |
+
networks = []
|
| 97 |
+
for i, lora_weight in enumerate(lora_weights or []):
|
| 98 |
+
multiplier = lora_multipliers[i] if lora_multipliers is not None and len(lora_multipliers) > i else 1.0
|
| 99 |
+
include = include_patterns[i] if include_patterns is not None and len(include_patterns) > i else None
|
| 100 |
+
exclude = exclude_patterns[i] if exclude_patterns is not None and len(exclude_patterns) > i else None
|
| 101 |
+
logger.info(f"Attaching LoRA weights from {lora_weight} with multiplier {multiplier}")
|
| 102 |
+
weights_sd = load_file(lora_weight)
|
| 103 |
+
if converter is not None:
|
| 104 |
+
weights_sd = converter(weights_sd)
|
| 105 |
+
weights_sd = filter_lora_state_dict(weights_sd, include, exclude)
|
| 106 |
+
network = lora_module.create_arch_network_from_weights(multiplier, weights_sd, unet=model, for_inference=True)
|
| 107 |
+
if not network.unet_loras:
|
| 108 |
+
raise ValueError(f"LoRA {lora_weight} contains no modules that match the model")
|
| 109 |
+
network.apply_to(None, model, apply_text_encoder=False, apply_unet=True)
|
| 110 |
+
network.load_state_dict(weights_sd, strict=True)
|
| 111 |
+
network.eval().requires_grad_(False).to(device)
|
| 112 |
+
networks.append(network)
|
| 113 |
+
return networks
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
def load_safetensors_with_lora_and_fp8(
|
| 117 |
+
model_files: Union[str, List[str]],
|
| 118 |
+
lora_weights_list: Optional[List[Dict[str, torch.Tensor]]],
|
| 119 |
+
lora_multipliers: Optional[List[float]],
|
| 120 |
+
fp8_optimization: bool,
|
| 121 |
+
calc_device: torch.device,
|
| 122 |
+
move_to_device: bool = False,
|
| 123 |
+
dit_weight_dtype: Optional[torch.dtype] = None,
|
| 124 |
+
target_keys: Optional[List[str]] = None,
|
| 125 |
+
exclude_keys: Optional[List[str]] = None,
|
| 126 |
+
disable_numpy_memmap: bool = False,
|
| 127 |
+
weight_transform_hooks: Optional[WeightTransformHooks] = None,
|
| 128 |
+
allow_prequantized_fp8: bool = False,
|
| 129 |
+
quantizer=None,
|
| 130 |
+
) -> dict[str, torch.Tensor]:
|
| 131 |
+
"""
|
| 132 |
+
Merge LoRA weights into the state dict of a model with fp8 optimization if needed.
|
| 133 |
+
|
| 134 |
+
Args:
|
| 135 |
+
model_files (Union[str, List[str]]): Path to the model file or list of paths. If the path matches a pattern like `00001-of-00004`, it will load all files with the same prefix.
|
| 136 |
+
lora_weights_list (Optional[List[Dict[str, torch.Tensor]]]): List of dictionaries of LoRA weight tensors to load.
|
| 137 |
+
lora_multipliers (Optional[List[float]]): List of multipliers for LoRA weights.
|
| 138 |
+
fp8_optimization (bool): Whether to apply FP8 optimization.
|
| 139 |
+
calc_device (torch.device): Device to calculate on.
|
| 140 |
+
move_to_device (bool): Whether to move tensors to the calculation device after loading.
|
| 141 |
+
target_keys (Optional[List[str]]): Keys to target for optimization.
|
| 142 |
+
exclude_keys (Optional[List[str]]): Keys to exclude from optimization.
|
| 143 |
+
disable_numpy_memmap (bool): Whether to disable numpy memmap when loading safetensors.
|
| 144 |
+
weight_transform_hooks (Optional[WeightTransformHooks]): Hooks for transforming weights during loading.
|
| 145 |
+
quantizer: Optional quantization strategy object with its own streaming loader
|
| 146 |
+
(e.g. ConvRotInt8Quantizer). Mutually exclusive with fp8_optimization. The LoRA
|
| 147 |
+
merge weight_hook is passed through, so LoRA is merged before quantization.
|
| 148 |
+
"""
|
| 149 |
+
|
| 150 |
+
# if the file name ends with 00001-of-00004 etc, we need to load the files with the same prefix
|
| 151 |
+
if isinstance(model_files, str):
|
| 152 |
+
model_files = [model_files]
|
| 153 |
+
|
| 154 |
+
extended_model_files = []
|
| 155 |
+
for model_file in model_files:
|
| 156 |
+
split_filenames = get_split_weight_filenames(model_file)
|
| 157 |
+
if split_filenames is not None:
|
| 158 |
+
extended_model_files.extend(split_filenames)
|
| 159 |
+
else:
|
| 160 |
+
extended_model_files.append(model_file)
|
| 161 |
+
model_files = extended_model_files
|
| 162 |
+
logger.info(f"Loading model files: {model_files}")
|
| 163 |
+
|
| 164 |
+
# load LoRA weights
|
| 165 |
+
weight_hook = None
|
| 166 |
+
if lora_weights_list is None or len(lora_weights_list) == 0:
|
| 167 |
+
lora_weights_list = []
|
| 168 |
+
lora_multipliers = []
|
| 169 |
+
list_of_lora_weight_keys = []
|
| 170 |
+
else:
|
| 171 |
+
list_of_lora_weight_keys = []
|
| 172 |
+
for lora_sd in lora_weights_list:
|
| 173 |
+
lora_weight_keys = set(lora_sd.keys())
|
| 174 |
+
list_of_lora_weight_keys.append(lora_weight_keys)
|
| 175 |
+
|
| 176 |
+
if lora_multipliers is None:
|
| 177 |
+
lora_multipliers = [1.0] * len(lora_weights_list)
|
| 178 |
+
while len(lora_multipliers) < len(lora_weights_list):
|
| 179 |
+
lora_multipliers.append(1.0)
|
| 180 |
+
if len(lora_multipliers) > len(lora_weights_list):
|
| 181 |
+
lora_multipliers = lora_multipliers[: len(lora_weights_list)]
|
| 182 |
+
|
| 183 |
+
# detect network types for each lora_sd
|
| 184 |
+
lora_network_types = [detect_network_type(lora_sd) for lora_sd in lora_weights_list]
|
| 185 |
+
|
| 186 |
+
# Merge LoRA weights into the state dict
|
| 187 |
+
logger.info(f"Merging LoRA weights into state dict. multipliers: {lora_multipliers}, network types: {lora_network_types}")
|
| 188 |
+
|
| 189 |
+
# make hook for LoRA merging
|
| 190 |
+
def weight_hook_func(model_weight_key, model_weight: torch.Tensor, keep_on_calc_device=False):
|
| 191 |
+
nonlocal list_of_lora_weight_keys, lora_weights_list, lora_multipliers, lora_network_types, calc_device
|
| 192 |
+
|
| 193 |
+
if not model_weight_key.endswith(".weight"):
|
| 194 |
+
return model_weight
|
| 195 |
+
|
| 196 |
+
original_device = model_weight.device
|
| 197 |
+
if original_device != calc_device:
|
| 198 |
+
model_weight = model_weight.to(calc_device) # to make calculation faster
|
| 199 |
+
|
| 200 |
+
for lora_weight_keys, lora_sd, multiplier, net_type in zip(
|
| 201 |
+
list_of_lora_weight_keys, lora_weights_list, lora_multipliers, lora_network_types
|
| 202 |
+
):
|
| 203 |
+
lora_name = model_weight_key.rsplit(".", 1)[0] # remove trailing ".weight"
|
| 204 |
+
lora_name = "lora_unet_" + lora_name.replace(".", "_")
|
| 205 |
+
|
| 206 |
+
if net_type == "loha":
|
| 207 |
+
from musubi_tuner.networks.loha import merge_weights_to_tensor as loha_merge
|
| 208 |
+
|
| 209 |
+
model_weight = loha_merge(model_weight, lora_name, lora_sd, lora_weight_keys, multiplier, calc_device)
|
| 210 |
+
elif net_type == "lokr":
|
| 211 |
+
from musubi_tuner.networks.lokr import merge_weights_to_tensor as lokr_merge
|
| 212 |
+
|
| 213 |
+
model_weight = lokr_merge(model_weight, lora_name, lora_sd, lora_weight_keys, multiplier, calc_device)
|
| 214 |
+
else:
|
| 215 |
+
# standard LoRA (lora_down/lora_up)
|
| 216 |
+
down_key = lora_name + ".lora_down.weight"
|
| 217 |
+
up_key = lora_name + ".lora_up.weight"
|
| 218 |
+
alpha_key = lora_name + ".alpha"
|
| 219 |
+
if down_key not in lora_weight_keys or up_key not in lora_weight_keys:
|
| 220 |
+
continue
|
| 221 |
+
|
| 222 |
+
# get LoRA weights
|
| 223 |
+
down_weight = lora_sd[down_key]
|
| 224 |
+
up_weight = lora_sd[up_key]
|
| 225 |
+
|
| 226 |
+
dim = down_weight.size()[0]
|
| 227 |
+
alpha = lora_sd.get(alpha_key, dim)
|
| 228 |
+
scale = alpha / dim
|
| 229 |
+
|
| 230 |
+
down_weight = down_weight.to(calc_device)
|
| 231 |
+
up_weight = up_weight.to(calc_device)
|
| 232 |
+
|
| 233 |
+
original_dtype = model_weight.dtype
|
| 234 |
+
if original_dtype.itemsize == 1: # fp8
|
| 235 |
+
# temporarily convert to float16 for calculation
|
| 236 |
+
model_weight = model_weight.to(torch.float16)
|
| 237 |
+
down_weight = down_weight.to(torch.float16)
|
| 238 |
+
up_weight = up_weight.to(torch.float16)
|
| 239 |
+
|
| 240 |
+
# W <- W + U * D
|
| 241 |
+
if len(model_weight.size()) == 2:
|
| 242 |
+
# linear
|
| 243 |
+
if len(up_weight.size()) == 4: # use linear projection mismatch
|
| 244 |
+
up_weight = up_weight.squeeze(3).squeeze(2)
|
| 245 |
+
down_weight = down_weight.squeeze(3).squeeze(2)
|
| 246 |
+
model_weight = model_weight + multiplier * (up_weight @ down_weight) * scale
|
| 247 |
+
elif down_weight.size()[2:4] == (1, 1):
|
| 248 |
+
# conv2d 1x1
|
| 249 |
+
model_weight = (
|
| 250 |
+
model_weight
|
| 251 |
+
+ multiplier
|
| 252 |
+
* (up_weight.squeeze(3).squeeze(2) @ down_weight.squeeze(3).squeeze(2)).unsqueeze(2).unsqueeze(3)
|
| 253 |
+
* scale
|
| 254 |
+
)
|
| 255 |
+
else:
|
| 256 |
+
# conv2d 3x3
|
| 257 |
+
conved = torch.nn.functional.conv2d(down_weight.permute(1, 0, 2, 3), up_weight).permute(1, 0, 2, 3)
|
| 258 |
+
model_weight = model_weight + multiplier * conved * scale
|
| 259 |
+
|
| 260 |
+
if original_dtype.itemsize == 1: # fp8
|
| 261 |
+
model_weight = model_weight.to(original_dtype) # convert back to original dtype
|
| 262 |
+
|
| 263 |
+
# remove LoRA keys from set
|
| 264 |
+
lora_weight_keys.remove(down_key)
|
| 265 |
+
lora_weight_keys.remove(up_key)
|
| 266 |
+
if alpha_key in lora_weight_keys:
|
| 267 |
+
lora_weight_keys.remove(alpha_key)
|
| 268 |
+
|
| 269 |
+
if not keep_on_calc_device and original_device != calc_device:
|
| 270 |
+
model_weight = model_weight.to(original_device) # move back to original device
|
| 271 |
+
return model_weight
|
| 272 |
+
|
| 273 |
+
weight_hook = weight_hook_func
|
| 274 |
+
|
| 275 |
+
state_dict = load_safetensors_with_fp8_optimization_and_hook(
|
| 276 |
+
model_files,
|
| 277 |
+
fp8_optimization,
|
| 278 |
+
calc_device,
|
| 279 |
+
move_to_device,
|
| 280 |
+
dit_weight_dtype,
|
| 281 |
+
target_keys,
|
| 282 |
+
exclude_keys,
|
| 283 |
+
weight_hook=weight_hook,
|
| 284 |
+
disable_numpy_memmap=disable_numpy_memmap,
|
| 285 |
+
weight_transform_hooks=weight_transform_hooks,
|
| 286 |
+
allow_prequantized_fp8=allow_prequantized_fp8,
|
| 287 |
+
quantizer=quantizer,
|
| 288 |
+
)
|
| 289 |
+
|
| 290 |
+
for lora_weight_keys in list_of_lora_weight_keys:
|
| 291 |
+
# check if all LoRA keys are used
|
| 292 |
+
if len(lora_weight_keys) > 0:
|
| 293 |
+
# if there are still LoRA keys left, it means they are not used in the model
|
| 294 |
+
# this is a warning, not an error
|
| 295 |
+
logger.warning(f"Warning: not all LoRA keys are used: {', '.join(lora_weight_keys)}")
|
| 296 |
+
|
| 297 |
+
return state_dict
|
| 298 |
+
|
| 299 |
+
|
| 300 |
+
def load_safetensors_with_fp8_optimization_and_hook(
|
| 301 |
+
model_files: list[str],
|
| 302 |
+
fp8_optimization: bool,
|
| 303 |
+
calc_device: torch.device,
|
| 304 |
+
move_to_device: bool = False,
|
| 305 |
+
dit_weight_dtype: Optional[torch.dtype] = None,
|
| 306 |
+
target_keys: Optional[List[str]] = None,
|
| 307 |
+
exclude_keys: Optional[List[str]] = None,
|
| 308 |
+
weight_hook: callable = None,
|
| 309 |
+
disable_numpy_memmap: bool = False,
|
| 310 |
+
weight_transform_hooks: Optional[WeightTransformHooks] = None,
|
| 311 |
+
allow_prequantized_fp8: bool = False,
|
| 312 |
+
quantizer=None,
|
| 313 |
+
) -> dict[str, torch.Tensor]:
|
| 314 |
+
"""
|
| 315 |
+
Load state dict from safetensors files and merge LoRA weights into the state dict with fp8 optimization if needed.
|
| 316 |
+
"""
|
| 317 |
+
if quantizer is not None:
|
| 318 |
+
assert not fp8_optimization, "quantizer and fp8_optimization are mutually exclusive"
|
| 319 |
+
logger.info(f"Loading state dict with {type(quantizer).__name__}. Hook enabled: {weight_hook is not None}")
|
| 320 |
+
return quantizer.load_and_quantize(
|
| 321 |
+
model_files,
|
| 322 |
+
calc_device,
|
| 323 |
+
move_to_device=move_to_device,
|
| 324 |
+
weight_hook=weight_hook,
|
| 325 |
+
disable_numpy_memmap=disable_numpy_memmap,
|
| 326 |
+
weight_transform_hooks=weight_transform_hooks,
|
| 327 |
+
)
|
| 328 |
+
if fp8_optimization:
|
| 329 |
+
logger.info(
|
| 330 |
+
f"Loading state dict with FP8 optimization. Dtype of weight: {dit_weight_dtype}, hook enabled: {weight_hook is not None}"
|
| 331 |
+
)
|
| 332 |
+
# dit_weight_dtype is not used because we use fp8 optimization
|
| 333 |
+
state_dict = load_safetensors_with_fp8_optimization(
|
| 334 |
+
model_files,
|
| 335 |
+
calc_device,
|
| 336 |
+
target_keys,
|
| 337 |
+
exclude_keys,
|
| 338 |
+
move_to_device=move_to_device,
|
| 339 |
+
weight_hook=weight_hook,
|
| 340 |
+
disable_numpy_memmap=disable_numpy_memmap,
|
| 341 |
+
weight_transform_hooks=weight_transform_hooks,
|
| 342 |
+
allow_prequantized_fp8=allow_prequantized_fp8,
|
| 343 |
+
)
|
| 344 |
+
else:
|
| 345 |
+
logger.info(
|
| 346 |
+
f"Loading state dict without FP8 optimization. Dtype of weight: {dit_weight_dtype}, hook enabled: {weight_hook is not None}"
|
| 347 |
+
)
|
| 348 |
+
state_dict = {}
|
| 349 |
+
for model_file in model_files:
|
| 350 |
+
with MemoryEfficientSafeOpen(model_file, disable_numpy_memmap=disable_numpy_memmap) as original_f:
|
| 351 |
+
f = TensorWeightAdapter(weight_transform_hooks, original_f) if weight_transform_hooks is not None else original_f
|
| 352 |
+
for key in tqdm(f.keys(), desc=f"Loading {os.path.basename(model_file)}", leave=False):
|
| 353 |
+
if weight_hook is None and move_to_device:
|
| 354 |
+
value = f.get_tensor(key, device=calc_device, dtype=dit_weight_dtype)
|
| 355 |
+
else:
|
| 356 |
+
value = f.get_tensor(key) # we cannot directly load to device because get_tensor does non-blocking transfer
|
| 357 |
+
if weight_hook is not None:
|
| 358 |
+
value = weight_hook(key, value, keep_on_calc_device=move_to_device)
|
| 359 |
+
if move_to_device:
|
| 360 |
+
value = value.to(calc_device, dtype=dit_weight_dtype, non_blocking=True)
|
| 361 |
+
elif dit_weight_dtype is not None:
|
| 362 |
+
value = value.to(dit_weight_dtype)
|
| 363 |
+
|
| 364 |
+
state_dict[key] = value
|
| 365 |
+
if move_to_device:
|
| 366 |
+
synchronize_device(calc_device)
|
| 367 |
+
|
| 368 |
+
return state_dict
|
third_party/musubi_tuner/utils/model_utils.py
ADDED
|
@@ -0,0 +1,342 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import argparse
|
| 2 |
+
import hashlib
|
| 3 |
+
from functools import lru_cache
|
| 4 |
+
from io import BytesIO
|
| 5 |
+
from typing import Any, Callable, Optional
|
| 6 |
+
import logging
|
| 7 |
+
import safetensors.torch
|
| 8 |
+
import torch
|
| 9 |
+
|
| 10 |
+
logger = logging.getLogger(__name__)
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def model_hash(filename):
|
| 14 |
+
"""Old model hash used by stable-diffusion-webui"""
|
| 15 |
+
try:
|
| 16 |
+
with open(filename, "rb") as file:
|
| 17 |
+
m = hashlib.sha256()
|
| 18 |
+
|
| 19 |
+
file.seek(0x100000)
|
| 20 |
+
m.update(file.read(0x10000))
|
| 21 |
+
return m.hexdigest()[0:8]
|
| 22 |
+
except FileNotFoundError:
|
| 23 |
+
return "NOFILE"
|
| 24 |
+
except IsADirectoryError: # Linux?
|
| 25 |
+
return "IsADirectory"
|
| 26 |
+
except PermissionError: # Windows
|
| 27 |
+
return "IsADirectory"
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def calculate_sha256(filename):
|
| 31 |
+
"""New model hash used by stable-diffusion-webui"""
|
| 32 |
+
try:
|
| 33 |
+
hash_sha256 = hashlib.sha256()
|
| 34 |
+
blksize = 1024 * 1024
|
| 35 |
+
|
| 36 |
+
with open(filename, "rb") as f:
|
| 37 |
+
for chunk in iter(lambda: f.read(blksize), b""):
|
| 38 |
+
hash_sha256.update(chunk)
|
| 39 |
+
|
| 40 |
+
return hash_sha256.hexdigest()
|
| 41 |
+
except FileNotFoundError:
|
| 42 |
+
return "NOFILE"
|
| 43 |
+
except IsADirectoryError: # Linux?
|
| 44 |
+
return "IsADirectory"
|
| 45 |
+
except PermissionError: # Windows
|
| 46 |
+
return "IsADirectory"
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def addnet_hash_legacy(b):
|
| 50 |
+
"""Old model hash used by sd-webui-additional-networks for .safetensors format files"""
|
| 51 |
+
m = hashlib.sha256()
|
| 52 |
+
|
| 53 |
+
b.seek(0x100000)
|
| 54 |
+
m.update(b.read(0x10000))
|
| 55 |
+
return m.hexdigest()[0:8]
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def addnet_hash_safetensors(b):
|
| 59 |
+
"""New model hash used by sd-webui-additional-networks for .safetensors format files"""
|
| 60 |
+
hash_sha256 = hashlib.sha256()
|
| 61 |
+
blksize = 1024 * 1024
|
| 62 |
+
|
| 63 |
+
b.seek(0)
|
| 64 |
+
header = b.read(8)
|
| 65 |
+
n = int.from_bytes(header, "little")
|
| 66 |
+
|
| 67 |
+
offset = n + 8
|
| 68 |
+
b.seek(offset)
|
| 69 |
+
for chunk in iter(lambda: b.read(blksize), b""):
|
| 70 |
+
hash_sha256.update(chunk)
|
| 71 |
+
|
| 72 |
+
return hash_sha256.hexdigest()
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def precalculate_safetensors_hashes(tensors, metadata):
|
| 76 |
+
"""Precalculate the model hashes needed by sd-webui-additional-networks to
|
| 77 |
+
save time on indexing the model later."""
|
| 78 |
+
|
| 79 |
+
# Because writing user metadata to the file can change the result of
|
| 80 |
+
# sd_models.model_hash(), only retain the training metadata for purposes of
|
| 81 |
+
# calculating the hash, as they are meant to be immutable
|
| 82 |
+
metadata = {k: v for k, v in metadata.items() if k.startswith("ss_")}
|
| 83 |
+
|
| 84 |
+
bytes = safetensors.torch.save(tensors, metadata)
|
| 85 |
+
b = BytesIO(bytes)
|
| 86 |
+
|
| 87 |
+
model_hash = addnet_hash_safetensors(b)
|
| 88 |
+
legacy_hash = addnet_hash_legacy(b)
|
| 89 |
+
return model_hash, legacy_hash
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def dtype_to_str(dtype: torch.dtype) -> str:
|
| 93 |
+
# get name of the dtype
|
| 94 |
+
dtype_name = str(dtype).split(".")[-1]
|
| 95 |
+
return dtype_name
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
def str_to_dtype(s: Optional[str], default_dtype: Optional[torch.dtype] = None) -> torch.dtype:
|
| 99 |
+
"""
|
| 100 |
+
Convert a string to a torch.dtype
|
| 101 |
+
|
| 102 |
+
Args:
|
| 103 |
+
s: string representation of the dtype
|
| 104 |
+
default_dtype: default dtype to return if s is None
|
| 105 |
+
|
| 106 |
+
Returns:
|
| 107 |
+
torch.dtype: the corresponding torch.dtype
|
| 108 |
+
|
| 109 |
+
Raises:
|
| 110 |
+
ValueError: if the dtype is not supported
|
| 111 |
+
|
| 112 |
+
Examples:
|
| 113 |
+
>>> str_to_dtype("float32")
|
| 114 |
+
torch.float32
|
| 115 |
+
>>> str_to_dtype("fp32")
|
| 116 |
+
torch.float32
|
| 117 |
+
>>> str_to_dtype("float16")
|
| 118 |
+
torch.float16
|
| 119 |
+
>>> str_to_dtype("fp16")
|
| 120 |
+
torch.float16
|
| 121 |
+
>>> str_to_dtype("bfloat16")
|
| 122 |
+
torch.bfloat16
|
| 123 |
+
>>> str_to_dtype("bf16")
|
| 124 |
+
torch.bfloat16
|
| 125 |
+
>>> str_to_dtype("fp8")
|
| 126 |
+
torch.float8_e4m3fn
|
| 127 |
+
>>> str_to_dtype("fp8_e4m3fn")
|
| 128 |
+
torch.float8_e4m3fn
|
| 129 |
+
>>> str_to_dtype("fp8_e4m3fnuz")
|
| 130 |
+
torch.float8_e4m3fnuz
|
| 131 |
+
>>> str_to_dtype("fp8_e5m2")
|
| 132 |
+
torch.float8_e5m2
|
| 133 |
+
>>> str_to_dtype("fp8_e5m2fnuz")
|
| 134 |
+
torch.float8_e5m2fnuz
|
| 135 |
+
"""
|
| 136 |
+
if s is None:
|
| 137 |
+
return default_dtype
|
| 138 |
+
if s in ["bf16", "bfloat16"]:
|
| 139 |
+
return torch.bfloat16
|
| 140 |
+
elif s in ["fp16", "float16"]:
|
| 141 |
+
return torch.float16
|
| 142 |
+
elif s in ["fp32", "float32", "float"]:
|
| 143 |
+
return torch.float32
|
| 144 |
+
elif s in ["fp8_e4m3fn", "e4m3fn", "float8_e4m3fn"]:
|
| 145 |
+
return torch.float8_e4m3fn
|
| 146 |
+
elif s in ["fp8_e4m3fnuz", "e4m3fnuz", "float8_e4m3fnuz"]:
|
| 147 |
+
return torch.float8_e4m3fnuz
|
| 148 |
+
elif s in ["fp8_e5m2", "e5m2", "float8_e5m2"]:
|
| 149 |
+
return torch.float8_e5m2
|
| 150 |
+
elif s in ["fp8_e5m2fnuz", "e5m2fnuz", "float8_e5m2fnuz"]:
|
| 151 |
+
return torch.float8_e5m2fnuz
|
| 152 |
+
elif s in ["fp8", "float8"]:
|
| 153 |
+
return torch.float8_e4m3fn # default fp8
|
| 154 |
+
else:
|
| 155 |
+
raise ValueError(f"Unsupported dtype: {s}")
|
| 156 |
+
|
| 157 |
+
|
| 158 |
+
@lru_cache(maxsize=1)
|
| 159 |
+
def _known_dtype_strs() -> tuple[str, ...]:
|
| 160 |
+
"""All dtype strings dtype_to_str() can emit, longest first.
|
| 161 |
+
|
| 162 |
+
Sorting by length keeps multi-underscore names (e.g. "float8_e4m3fn") ahead of their
|
| 163 |
+
prefixes ("float8...") so the longest matching suffix wins.
|
| 164 |
+
"""
|
| 165 |
+
names = set()
|
| 166 |
+
for attr in dir(torch):
|
| 167 |
+
try:
|
| 168 |
+
obj = getattr(torch, attr)
|
| 169 |
+
except Exception:
|
| 170 |
+
continue
|
| 171 |
+
if isinstance(obj, torch.dtype):
|
| 172 |
+
names.add(dtype_to_str(obj))
|
| 173 |
+
return tuple(sorted(names, key=len, reverse=True))
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
def remove_dtype_suffix(name: str) -> str:
|
| 177 |
+
"""Remove a trailing ``_<dtype>`` suffix (as written by dtype_to_str) from a cache key.
|
| 178 |
+
|
| 179 |
+
Robust to dtype names that contain underscores such as ``float8_e4m3fn``; a plain
|
| 180 |
+
``rsplit("_", 1)`` would only drop the final ``fn`` segment. Returns ``name`` unchanged
|
| 181 |
+
if it does not end with a known dtype suffix.
|
| 182 |
+
"""
|
| 183 |
+
for dtype_str in _known_dtype_strs():
|
| 184 |
+
suffix = "_" + dtype_str
|
| 185 |
+
if name.endswith(suffix):
|
| 186 |
+
return name[: -len(suffix)]
|
| 187 |
+
return name
|
| 188 |
+
|
| 189 |
+
|
| 190 |
+
def to_device(x: Any, device: torch.device) -> Any:
|
| 191 |
+
if isinstance(x, torch.Tensor):
|
| 192 |
+
return x.to(device)
|
| 193 |
+
elif isinstance(x, list):
|
| 194 |
+
return [to_device(elem, device) for elem in x]
|
| 195 |
+
elif isinstance(x, tuple):
|
| 196 |
+
return tuple(to_device(elem, device) for elem in x)
|
| 197 |
+
elif isinstance(x, dict):
|
| 198 |
+
return {k: to_device(v, device) for k, v in x.items()}
|
| 199 |
+
else:
|
| 200 |
+
return x
|
| 201 |
+
|
| 202 |
+
|
| 203 |
+
def to_cpu(x: Any) -> Any:
|
| 204 |
+
"""
|
| 205 |
+
Recursively moves torch.Tensor objects (and containers thereof) to CPU.
|
| 206 |
+
|
| 207 |
+
Args:
|
| 208 |
+
x: A torch.Tensor, or a (possibly nested) list, tuple, or dict containing tensors.
|
| 209 |
+
|
| 210 |
+
Returns:
|
| 211 |
+
The same structure as x, with all torch.Tensor objects moved to CPU.
|
| 212 |
+
Non-tensor objects are returned unchanged.
|
| 213 |
+
"""
|
| 214 |
+
if isinstance(x, torch.Tensor):
|
| 215 |
+
return x.cpu()
|
| 216 |
+
elif isinstance(x, list):
|
| 217 |
+
return [to_cpu(elem) for elem in x]
|
| 218 |
+
elif isinstance(x, tuple):
|
| 219 |
+
return tuple(to_cpu(elem) for elem in x)
|
| 220 |
+
elif isinstance(x, dict):
|
| 221 |
+
return {k: to_cpu(v) for k, v in x.items()}
|
| 222 |
+
else:
|
| 223 |
+
return x
|
| 224 |
+
|
| 225 |
+
|
| 226 |
+
def create_cpu_offloading_wrapper(func: Callable, device: torch.device) -> Callable:
|
| 227 |
+
"""
|
| 228 |
+
Create a wrapper function that offloads inputs to CPU before calling the original function
|
| 229 |
+
and moves outputs back to the specified device.
|
| 230 |
+
|
| 231 |
+
Args:
|
| 232 |
+
func: The original function to wrap.
|
| 233 |
+
device: The device to move outputs back to.
|
| 234 |
+
|
| 235 |
+
Returns:
|
| 236 |
+
A wrapped function that offloads inputs to CPU and moves outputs back to the specified device.
|
| 237 |
+
"""
|
| 238 |
+
|
| 239 |
+
def wrapper(orig_func: Callable) -> Callable:
|
| 240 |
+
def custom_forward(*inputs):
|
| 241 |
+
nonlocal device, orig_func
|
| 242 |
+
cuda_inputs = to_device(inputs, device)
|
| 243 |
+
outputs = orig_func(*cuda_inputs)
|
| 244 |
+
return to_cpu(outputs)
|
| 245 |
+
|
| 246 |
+
return custom_forward
|
| 247 |
+
|
| 248 |
+
return wrapper(func)
|
| 249 |
+
|
| 250 |
+
|
| 251 |
+
def disable_linear_from_compile(module: torch.nn.Module):
|
| 252 |
+
"""Monkey-patch to disable torch.compile for all Linear layers (if the class name ends with 'Linear') in the given module."""
|
| 253 |
+
for sub_module in module.modules():
|
| 254 |
+
# if isinstance(sub_module, torch.nn.Linear):
|
| 255 |
+
if sub_module.__class__.__name__.endswith("Linear"):
|
| 256 |
+
if not hasattr(sub_module, "_forward_before_disable_compile"):
|
| 257 |
+
sub_module._forward_before_disable_compile = sub_module.forward
|
| 258 |
+
sub_module._eager_forward = torch._dynamo.disable()(sub_module.forward)
|
| 259 |
+
sub_module.forward = sub_module._eager_forward # override forward to disable compile
|
| 260 |
+
|
| 261 |
+
|
| 262 |
+
def setup_parser_compile(parser: argparse.ArgumentParser) -> None:
|
| 263 |
+
"""Add the torch.compile argument set consumed by compile_transformer below.
|
| 264 |
+
|
| 265 |
+
Same arguments as hv_generate_video.setup_parser_compile and the compile slice of
|
| 266 |
+
training's parser_common, placed here so scripts can add them without importing an
|
| 267 |
+
architecture module or the training-only dynamo/tf32 arguments.
|
| 268 |
+
"""
|
| 269 |
+
parser.add_argument(
|
| 270 |
+
"--compile",
|
| 271 |
+
action="store_true",
|
| 272 |
+
help="Enable torch.compile (requires Triton) / torch.compileを有効にする(Tritonが必要)",
|
| 273 |
+
)
|
| 274 |
+
parser.add_argument(
|
| 275 |
+
"--compile_backend",
|
| 276 |
+
type=str,
|
| 277 |
+
default="inductor",
|
| 278 |
+
help="torch.compile backend (default: inductor) / torch.compileのバックエンド(デフォルト: inductor)",
|
| 279 |
+
)
|
| 280 |
+
parser.add_argument(
|
| 281 |
+
"--compile_mode",
|
| 282 |
+
type=str,
|
| 283 |
+
default="default",
|
| 284 |
+
choices=["default", "reduce-overhead", "max-autotune", "max-autotune-no-cudagraphs"],
|
| 285 |
+
help="torch.compile mode (default: default) / torch.compileのモード(デフォルト: default)",
|
| 286 |
+
)
|
| 287 |
+
parser.add_argument(
|
| 288 |
+
"--compile_dynamic",
|
| 289 |
+
type=str,
|
| 290 |
+
default=None,
|
| 291 |
+
choices=["true", "false", "auto"],
|
| 292 |
+
help="Dynamic shapes mode for torch.compile (default: None, same as auto)"
|
| 293 |
+
" / torch.compileの動的形状モード(デフォルト: None、autoと同じ動作)",
|
| 294 |
+
)
|
| 295 |
+
parser.add_argument(
|
| 296 |
+
"--compile_fullgraph",
|
| 297 |
+
action="store_true",
|
| 298 |
+
help="Enable fullgraph mode in torch.compile / torch.compileでフルグラフモードを有効にする",
|
| 299 |
+
)
|
| 300 |
+
parser.add_argument(
|
| 301 |
+
"--compile_cache_size_limit",
|
| 302 |
+
type=int,
|
| 303 |
+
default=None,
|
| 304 |
+
help="Set torch._dynamo.config.cache_size_limit (default: PyTorch default, typically 8-32)"
|
| 305 |
+
" / torch._dynamo.config.cache_size_limitを設定(デフォルト: PyTorchのデフォルト、通常8-32)",
|
| 306 |
+
)
|
| 307 |
+
|
| 308 |
+
|
| 309 |
+
def compile_transformer(
|
| 310 |
+
args: argparse.Namespace,
|
| 311 |
+
transformer: torch.nn.Module,
|
| 312 |
+
target_blocks: list[torch.nn.ModuleList | list[torch.nn.Module]],
|
| 313 |
+
disable_linear: bool,
|
| 314 |
+
) -> torch.nn.Module:
|
| 315 |
+
if disable_linear:
|
| 316 |
+
logger.info("Disable linear from torch.compile for swap blocks...")
|
| 317 |
+
for blocks in target_blocks:
|
| 318 |
+
for block in blocks:
|
| 319 |
+
disable_linear_from_compile(block)
|
| 320 |
+
|
| 321 |
+
compile_dynamic = None
|
| 322 |
+
if args.compile_dynamic is not None:
|
| 323 |
+
compile_dynamic = {"true": True, "false": False, "auto": None}[args.compile_dynamic.lower()]
|
| 324 |
+
|
| 325 |
+
logger.info(
|
| 326 |
+
f"Compiling DiT model with torch.compile: backend={args.compile_backend}, mode={args.compile_mode}, dynamic={compile_dynamic}, fullgraph={args.compile_fullgraph}"
|
| 327 |
+
)
|
| 328 |
+
|
| 329 |
+
if args.compile_cache_size_limit is not None:
|
| 330 |
+
torch._dynamo.config.cache_size_limit = args.compile_cache_size_limit
|
| 331 |
+
|
| 332 |
+
for blocks in target_blocks:
|
| 333 |
+
for i, block in enumerate(blocks):
|
| 334 |
+
block = torch.compile(
|
| 335 |
+
block,
|
| 336 |
+
backend=args.compile_backend,
|
| 337 |
+
mode=args.compile_mode,
|
| 338 |
+
dynamic=compile_dynamic,
|
| 339 |
+
fullgraph=args.compile_fullgraph,
|
| 340 |
+
)
|
| 341 |
+
blocks[i] = block
|
| 342 |
+
return transformer
|
third_party/musubi_tuner/utils/safetensors_utils.py
ADDED
|
@@ -0,0 +1,483 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from dataclasses import dataclass
|
| 2 |
+
import os
|
| 3 |
+
import re
|
| 4 |
+
import numpy as np
|
| 5 |
+
import torch
|
| 6 |
+
import json
|
| 7 |
+
import struct
|
| 8 |
+
from typing import Dict, Any, Union, Optional
|
| 9 |
+
|
| 10 |
+
from safetensors.torch import load_file
|
| 11 |
+
|
| 12 |
+
from musubi_tuner.utils.device_utils import synchronize_device
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def mem_eff_save_file(tensors: Dict[str, torch.Tensor], filename: str, metadata: Dict[str, Any] = None):
|
| 16 |
+
"""
|
| 17 |
+
memory efficient save file
|
| 18 |
+
"""
|
| 19 |
+
|
| 20 |
+
_TYPES = {
|
| 21 |
+
torch.float64: "F64",
|
| 22 |
+
torch.float32: "F32",
|
| 23 |
+
torch.float16: "F16",
|
| 24 |
+
torch.bfloat16: "BF16",
|
| 25 |
+
torch.int64: "I64",
|
| 26 |
+
torch.int32: "I32",
|
| 27 |
+
torch.int16: "I16",
|
| 28 |
+
torch.int8: "I8",
|
| 29 |
+
torch.uint8: "U8",
|
| 30 |
+
torch.bool: "BOOL",
|
| 31 |
+
getattr(torch, "float8_e5m2", None): "F8_E5M2",
|
| 32 |
+
getattr(torch, "float8_e4m3fn", None): "F8_E4M3",
|
| 33 |
+
}
|
| 34 |
+
_ALIGN = 256
|
| 35 |
+
|
| 36 |
+
def validate_metadata(metadata: Dict[str, Any]) -> Dict[str, str]:
|
| 37 |
+
validated = {}
|
| 38 |
+
for key, value in metadata.items():
|
| 39 |
+
if not isinstance(key, str):
|
| 40 |
+
raise ValueError(f"Metadata key must be a string, got {type(key)}")
|
| 41 |
+
if not isinstance(value, str):
|
| 42 |
+
print(f"Warning: Metadata value for key '{key}' is not a string. Converting to string.")
|
| 43 |
+
validated[key] = str(value)
|
| 44 |
+
else:
|
| 45 |
+
validated[key] = value
|
| 46 |
+
return validated
|
| 47 |
+
|
| 48 |
+
# print(f"Using memory efficient save file: {filename}")
|
| 49 |
+
|
| 50 |
+
header = {}
|
| 51 |
+
offset = 0
|
| 52 |
+
if metadata:
|
| 53 |
+
header["__metadata__"] = validate_metadata(metadata)
|
| 54 |
+
for k, v in tensors.items():
|
| 55 |
+
if v.numel() == 0: # empty tensor
|
| 56 |
+
header[k] = {"dtype": _TYPES[v.dtype], "shape": list(v.shape), "data_offsets": [offset, offset]}
|
| 57 |
+
else:
|
| 58 |
+
size = v.numel() * v.element_size()
|
| 59 |
+
header[k] = {"dtype": _TYPES[v.dtype], "shape": list(v.shape), "data_offsets": [offset, offset + size]}
|
| 60 |
+
offset += size
|
| 61 |
+
|
| 62 |
+
hjson = json.dumps(header).encode("utf-8")
|
| 63 |
+
hjson += b" " * (-(len(hjson) + 8) % _ALIGN)
|
| 64 |
+
|
| 65 |
+
with open(filename, "wb") as f:
|
| 66 |
+
f.write(struct.pack("<Q", len(hjson)))
|
| 67 |
+
f.write(hjson)
|
| 68 |
+
|
| 69 |
+
for k, v in tensors.items():
|
| 70 |
+
if v.numel() == 0:
|
| 71 |
+
continue
|
| 72 |
+
if v.is_cuda:
|
| 73 |
+
# Direct GPU to disk save
|
| 74 |
+
with torch.cuda.device(v.device):
|
| 75 |
+
if v.dim() == 0: # if scalar, need to add a dimension to work with view
|
| 76 |
+
v = v.unsqueeze(0)
|
| 77 |
+
tensor_bytes = v.contiguous().view(torch.uint8)
|
| 78 |
+
tensor_bytes.cpu().numpy().tofile(f)
|
| 79 |
+
else:
|
| 80 |
+
# CPU tensor save
|
| 81 |
+
if v.dim() == 0: # if scalar, need to add a dimension to work with view
|
| 82 |
+
v = v.unsqueeze(0)
|
| 83 |
+
v.contiguous().view(torch.uint8).numpy().tofile(f)
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
class MemoryEfficientSafeOpen:
|
| 87 |
+
"""Memory-efficient reader for safetensors files.
|
| 88 |
+
|
| 89 |
+
This class provides a memory-efficient way to read tensors from safetensors files
|
| 90 |
+
by using memory mapping for large tensors and avoiding unnecessary copies.
|
| 91 |
+
"""
|
| 92 |
+
|
| 93 |
+
def __init__(self, filename, disable_numpy_memmap=False):
|
| 94 |
+
"""Initialize the SafeTensor reader.
|
| 95 |
+
|
| 96 |
+
Args:
|
| 97 |
+
filename (str): Path to the safetensors file to read.
|
| 98 |
+
disable_numpy_memmap (bool): If True, disable numpy memory mapping for large tensors, using standard file read instead.
|
| 99 |
+
"""
|
| 100 |
+
self.filename = filename
|
| 101 |
+
self.file = open(filename, "rb")
|
| 102 |
+
self.header, self.header_size = self._read_header()
|
| 103 |
+
self.disable_numpy_memmap = disable_numpy_memmap
|
| 104 |
+
|
| 105 |
+
def __enter__(self):
|
| 106 |
+
"""Enter context manager."""
|
| 107 |
+
return self
|
| 108 |
+
|
| 109 |
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
| 110 |
+
"""Exit context manager and close file."""
|
| 111 |
+
self.file.close()
|
| 112 |
+
|
| 113 |
+
def keys(self):
|
| 114 |
+
"""Get all tensor keys in the file.
|
| 115 |
+
|
| 116 |
+
Returns:
|
| 117 |
+
list: List of tensor names (excludes metadata).
|
| 118 |
+
"""
|
| 119 |
+
return [k for k in self.header.keys() if k != "__metadata__"]
|
| 120 |
+
|
| 121 |
+
def metadata(self) -> Dict[str, str]:
|
| 122 |
+
"""Get metadata from the file.
|
| 123 |
+
|
| 124 |
+
Returns:
|
| 125 |
+
Dict[str, str]: Metadata dictionary.
|
| 126 |
+
"""
|
| 127 |
+
return self.header.get("__metadata__", {})
|
| 128 |
+
|
| 129 |
+
def _read_header(self):
|
| 130 |
+
"""Read and parse the header from the safetensors file.
|
| 131 |
+
|
| 132 |
+
Returns:
|
| 133 |
+
tuple: (header_dict, header_size) containing parsed header and its size.
|
| 134 |
+
"""
|
| 135 |
+
# Read header size (8 bytes, little-endian unsigned long long)
|
| 136 |
+
header_size = struct.unpack("<Q", self.file.read(8))[0]
|
| 137 |
+
# Read and decode header JSON
|
| 138 |
+
header_json = self.file.read(header_size).decode("utf-8")
|
| 139 |
+
return json.loads(header_json), header_size
|
| 140 |
+
|
| 141 |
+
def get_tensor(self, key: str, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None):
|
| 142 |
+
"""Load a tensor from the file with memory-efficient strategies.
|
| 143 |
+
|
| 144 |
+
**Note:**
|
| 145 |
+
If device is 'cuda' , the transfer to GPU is done efficiently using pinned memory and non-blocking transfer.
|
| 146 |
+
So you must ensure that the transfer is completed before using the tensor (e.g., by `torch.cuda.synchronize()`).
|
| 147 |
+
|
| 148 |
+
If the tensor is large (>10MB) and the target device is CUDA, memory mapping with numpy.memmap is used to avoid intermediate copies.
|
| 149 |
+
|
| 150 |
+
Args:
|
| 151 |
+
key (str): Name of the tensor to load.
|
| 152 |
+
device (Optional[torch.device]): Target device for the tensor.
|
| 153 |
+
dtype (Optional[torch.dtype]): Target dtype for the tensor.
|
| 154 |
+
|
| 155 |
+
Returns:
|
| 156 |
+
torch.Tensor: The loaded tensor.
|
| 157 |
+
|
| 158 |
+
Raises:
|
| 159 |
+
KeyError: If the tensor key is not found in the file.
|
| 160 |
+
"""
|
| 161 |
+
if key not in self.header:
|
| 162 |
+
raise KeyError(f"Tensor '{key}' not found in the file")
|
| 163 |
+
|
| 164 |
+
metadata = self.header[key]
|
| 165 |
+
offset_start, offset_end = metadata["data_offsets"]
|
| 166 |
+
num_bytes = offset_end - offset_start
|
| 167 |
+
|
| 168 |
+
original_dtype = self._get_torch_dtype(metadata["dtype"])
|
| 169 |
+
target_dtype = dtype if dtype is not None else original_dtype
|
| 170 |
+
|
| 171 |
+
# Handle empty tensors
|
| 172 |
+
if num_bytes == 0:
|
| 173 |
+
return torch.empty(metadata["shape"], dtype=target_dtype, device=device)
|
| 174 |
+
|
| 175 |
+
# Determine if we should use pinned memory for GPU transfer
|
| 176 |
+
non_blocking = device is not None and device.type == "cuda"
|
| 177 |
+
|
| 178 |
+
# Calculate absolute file offset
|
| 179 |
+
tensor_offset = self.header_size + 8 + offset_start # adjust offset by header size
|
| 180 |
+
|
| 181 |
+
# Memory mapping strategy for large tensors to GPU
|
| 182 |
+
# Use memmap for large tensors to avoid intermediate copies.
|
| 183 |
+
# If device is cpu, tensor is not copied to gpu, so using memmap locks the file, which is not desired.
|
| 184 |
+
# So we only use memmap if device is not cpu.
|
| 185 |
+
# If disable_numpy_memmap is True, skip numpy memory mapping to load with standard file read.
|
| 186 |
+
if not self.disable_numpy_memmap and num_bytes > 10 * 1024 * 1024 and device is not None and device.type != "cpu":
|
| 187 |
+
# Create memory map for zero-copy reading
|
| 188 |
+
mm = np.memmap(self.filename, mode="c", dtype=np.uint8, offset=tensor_offset, shape=(num_bytes,))
|
| 189 |
+
byte_tensor = torch.from_numpy(mm) # zero copy
|
| 190 |
+
del mm
|
| 191 |
+
|
| 192 |
+
# Deserialize tensor (view and reshape)
|
| 193 |
+
cpu_tensor = self._deserialize_tensor(byte_tensor, metadata) # view and reshape
|
| 194 |
+
del byte_tensor
|
| 195 |
+
|
| 196 |
+
# Transfer to target device and dtype
|
| 197 |
+
gpu_tensor = cpu_tensor.to(device=device, dtype=target_dtype, non_blocking=non_blocking)
|
| 198 |
+
del cpu_tensor
|
| 199 |
+
return gpu_tensor
|
| 200 |
+
|
| 201 |
+
# Standard file reading strategy for smaller tensors or CPU target
|
| 202 |
+
# seek to the specified position
|
| 203 |
+
self.file.seek(tensor_offset)
|
| 204 |
+
|
| 205 |
+
# read directly into a numpy array by numpy.fromfile without intermediate copy
|
| 206 |
+
numpy_array = np.fromfile(self.file, dtype=np.uint8, count=num_bytes)
|
| 207 |
+
byte_tensor = torch.from_numpy(numpy_array)
|
| 208 |
+
del numpy_array
|
| 209 |
+
|
| 210 |
+
# deserialize (view and reshape)
|
| 211 |
+
deserialized_tensor = self._deserialize_tensor(byte_tensor, metadata)
|
| 212 |
+
del byte_tensor
|
| 213 |
+
|
| 214 |
+
# cast to target dtype and move to device
|
| 215 |
+
return deserialized_tensor.to(device=device, dtype=target_dtype, non_blocking=non_blocking)
|
| 216 |
+
|
| 217 |
+
def _deserialize_tensor(self, byte_tensor: torch.Tensor, metadata: Dict):
|
| 218 |
+
"""Deserialize byte tensor to the correct shape and dtype.
|
| 219 |
+
|
| 220 |
+
Args:
|
| 221 |
+
byte_tensor (torch.Tensor): Raw byte tensor from file.
|
| 222 |
+
metadata (Dict): Tensor metadata containing dtype and shape info.
|
| 223 |
+
|
| 224 |
+
Returns:
|
| 225 |
+
torch.Tensor: Deserialized tensor with correct shape and dtype.
|
| 226 |
+
"""
|
| 227 |
+
dtype = self._get_torch_dtype(metadata["dtype"])
|
| 228 |
+
shape = metadata["shape"]
|
| 229 |
+
|
| 230 |
+
# Handle special float8 types
|
| 231 |
+
if metadata["dtype"] in ["F8_E5M2", "F8_E4M3"]:
|
| 232 |
+
return self._convert_float8(byte_tensor, metadata["dtype"], shape)
|
| 233 |
+
|
| 234 |
+
# Standard conversion: view as target dtype and reshape
|
| 235 |
+
return byte_tensor.view(dtype).reshape(shape)
|
| 236 |
+
|
| 237 |
+
@staticmethod
|
| 238 |
+
def _get_torch_dtype(dtype_str):
|
| 239 |
+
"""Convert string dtype to PyTorch dtype.
|
| 240 |
+
|
| 241 |
+
Args:
|
| 242 |
+
dtype_str (str): String representation of the dtype.
|
| 243 |
+
|
| 244 |
+
Returns:
|
| 245 |
+
torch.dtype: Corresponding PyTorch dtype.
|
| 246 |
+
"""
|
| 247 |
+
# Standard dtype mappings
|
| 248 |
+
dtype_map = {
|
| 249 |
+
"F64": torch.float64,
|
| 250 |
+
"F32": torch.float32,
|
| 251 |
+
"F16": torch.float16,
|
| 252 |
+
"BF16": torch.bfloat16,
|
| 253 |
+
"I64": torch.int64,
|
| 254 |
+
"I32": torch.int32,
|
| 255 |
+
"I16": torch.int16,
|
| 256 |
+
"I8": torch.int8,
|
| 257 |
+
"U8": torch.uint8,
|
| 258 |
+
"BOOL": torch.bool,
|
| 259 |
+
}
|
| 260 |
+
# Add float8 types if available in PyTorch version
|
| 261 |
+
if hasattr(torch, "float8_e5m2"):
|
| 262 |
+
dtype_map["F8_E5M2"] = torch.float8_e5m2
|
| 263 |
+
if hasattr(torch, "float8_e4m3fn"):
|
| 264 |
+
dtype_map["F8_E4M3"] = torch.float8_e4m3fn
|
| 265 |
+
return dtype_map.get(dtype_str)
|
| 266 |
+
|
| 267 |
+
@staticmethod
|
| 268 |
+
def _convert_float8(byte_tensor, dtype_str, shape):
|
| 269 |
+
"""Convert byte tensor to float8 format if supported.
|
| 270 |
+
|
| 271 |
+
Args:
|
| 272 |
+
byte_tensor (torch.Tensor): Raw byte tensor.
|
| 273 |
+
dtype_str (str): Float8 dtype string ("F8_E5M2" or "F8_E4M3").
|
| 274 |
+
shape (tuple): Target tensor shape.
|
| 275 |
+
|
| 276 |
+
Returns:
|
| 277 |
+
torch.Tensor: Tensor with float8 dtype.
|
| 278 |
+
|
| 279 |
+
Raises:
|
| 280 |
+
ValueError: If float8 type is not supported in current PyTorch version.
|
| 281 |
+
"""
|
| 282 |
+
# Convert to specific float8 types if available
|
| 283 |
+
if dtype_str == "F8_E5M2" and hasattr(torch, "float8_e5m2"):
|
| 284 |
+
return byte_tensor.view(torch.float8_e5m2).reshape(shape)
|
| 285 |
+
elif dtype_str == "F8_E4M3" and hasattr(torch, "float8_e4m3fn"):
|
| 286 |
+
return byte_tensor.view(torch.float8_e4m3fn).reshape(shape)
|
| 287 |
+
else:
|
| 288 |
+
# Float8 not supported in this PyTorch version
|
| 289 |
+
raise ValueError(f"Unsupported float8 type: {dtype_str} (upgrade PyTorch to support float8 types)")
|
| 290 |
+
|
| 291 |
+
|
| 292 |
+
def load_safetensors(
|
| 293 |
+
path: str,
|
| 294 |
+
device: Union[str, torch.device],
|
| 295 |
+
disable_mmap: bool = False,
|
| 296 |
+
dtype: Optional[torch.dtype] = None,
|
| 297 |
+
disable_numpy_memmap: bool = False,
|
| 298 |
+
) -> dict[str, torch.Tensor]:
|
| 299 |
+
if disable_mmap:
|
| 300 |
+
# return safetensors.torch.load(open(path, "rb").read())
|
| 301 |
+
# use experimental loader
|
| 302 |
+
# logger.info(f"Loading without mmap (experimental)")
|
| 303 |
+
state_dict = {}
|
| 304 |
+
device = torch.device(device) if device is not None else None
|
| 305 |
+
with MemoryEfficientSafeOpen(path, disable_numpy_memmap=disable_numpy_memmap) as f:
|
| 306 |
+
for key in f.keys():
|
| 307 |
+
state_dict[key] = f.get_tensor(key, device=device, dtype=dtype)
|
| 308 |
+
synchronize_device(device)
|
| 309 |
+
return state_dict
|
| 310 |
+
else:
|
| 311 |
+
try:
|
| 312 |
+
state_dict = load_file(path, device=device)
|
| 313 |
+
except:
|
| 314 |
+
state_dict = load_file(path) # prevent device invalid Error
|
| 315 |
+
if dtype is not None:
|
| 316 |
+
for key in state_dict.keys():
|
| 317 |
+
state_dict[key] = state_dict[key].to(dtype=dtype)
|
| 318 |
+
return state_dict
|
| 319 |
+
|
| 320 |
+
|
| 321 |
+
def get_split_weight_filenames(file_path: str) -> Optional[list[str]]:
|
| 322 |
+
"""
|
| 323 |
+
Get the list of split weight filenames (full paths) if the file name ends with 00001-of-00004 etc.
|
| 324 |
+
Returns None if the file is not split.
|
| 325 |
+
"""
|
| 326 |
+
basename = os.path.basename(file_path)
|
| 327 |
+
match = re.match(r"^(.*?)(\d+)-of-(\d+)\.safetensors$", basename)
|
| 328 |
+
if match:
|
| 329 |
+
prefix = basename[: match.start(2)]
|
| 330 |
+
count = int(match.group(3))
|
| 331 |
+
filenames = []
|
| 332 |
+
for i in range(count):
|
| 333 |
+
filename = f"{prefix}{i + 1:05d}-of-{count:05d}.safetensors"
|
| 334 |
+
filepath = os.path.join(os.path.dirname(file_path), filename)
|
| 335 |
+
if os.path.exists(filepath):
|
| 336 |
+
filenames.append(filepath)
|
| 337 |
+
else:
|
| 338 |
+
raise FileNotFoundError(f"File {filepath} not found")
|
| 339 |
+
return filenames
|
| 340 |
+
else:
|
| 341 |
+
return None
|
| 342 |
+
|
| 343 |
+
|
| 344 |
+
def load_split_weights(
|
| 345 |
+
file_path: str, device: Union[str, torch.device] = "cpu", disable_mmap: bool = False, dtype: Optional[torch.dtype] = None
|
| 346 |
+
) -> Dict[str, torch.Tensor]:
|
| 347 |
+
"""
|
| 348 |
+
Load split weights from a file. If the file name ends with 00001-of-00004 etc, it will load all files with the same prefix.
|
| 349 |
+
dtype is as is, no conversion is done.
|
| 350 |
+
"""
|
| 351 |
+
device = torch.device(device)
|
| 352 |
+
|
| 353 |
+
# if the file name ends with 00001-of-00004 etc, we need to load the files with the same prefix
|
| 354 |
+
split_filenames = get_split_weight_filenames(file_path)
|
| 355 |
+
if split_filenames is not None:
|
| 356 |
+
state_dict = {}
|
| 357 |
+
for filename in split_filenames:
|
| 358 |
+
state_dict.update(load_safetensors(filename, device=device, disable_mmap=disable_mmap, dtype=dtype))
|
| 359 |
+
else:
|
| 360 |
+
state_dict = load_safetensors(file_path, device=device, disable_mmap=disable_mmap, dtype=dtype)
|
| 361 |
+
return state_dict
|
| 362 |
+
|
| 363 |
+
|
| 364 |
+
def find_key(safetensors_file: str, starts_with: Optional[str] = None, ends_with: Optional[str] = None) -> Optional[str]:
|
| 365 |
+
"""
|
| 366 |
+
Find a key in a safetensors file that starts with `starts_with` and ends with `ends_with`.
|
| 367 |
+
If `starts_with` is None, it will match any key.
|
| 368 |
+
If `ends_with` is None, it will match any key.
|
| 369 |
+
Returns the first matching key or None if no key matches.
|
| 370 |
+
"""
|
| 371 |
+
with MemoryEfficientSafeOpen(safetensors_file) as f:
|
| 372 |
+
for key in f.keys():
|
| 373 |
+
if (starts_with is None or key.startswith(starts_with)) and (ends_with is None or key.endswith(ends_with)):
|
| 374 |
+
return key
|
| 375 |
+
return None
|
| 376 |
+
|
| 377 |
+
|
| 378 |
+
def find_keys(safetensors_file: str, starts_with: Optional[str] = None, ends_with: Optional[str] = None) -> list[str]:
|
| 379 |
+
"""
|
| 380 |
+
Find all keys in a safetensors file that start with `starts_with` and end with `ends_with`.
|
| 381 |
+
If `starts_with` is None, it will match any key. If `ends_with` is None, it will match any key.
|
| 382 |
+
|
| 383 |
+
The matching keys are returned sorted, so callers get a deterministic order regardless of the
|
| 384 |
+
order the keys happen to be stored in the file header. This matters when the keys are folded into
|
| 385 |
+
a bucket key (e.g. ``latents_control_{i}``): otherwise two files holding the same tensors in a
|
| 386 |
+
different header order, or with the index<->shape association swapped, could end up in different
|
| 387 |
+
buckets or be wrongly batched together.
|
| 388 |
+
"""
|
| 389 |
+
with MemoryEfficientSafeOpen(safetensors_file) as f:
|
| 390 |
+
keys = [
|
| 391 |
+
key
|
| 392 |
+
for key in f.keys()
|
| 393 |
+
if (starts_with is None or key.startswith(starts_with)) and (ends_with is None or key.endswith(ends_with))
|
| 394 |
+
]
|
| 395 |
+
return sorted(keys)
|
| 396 |
+
|
| 397 |
+
|
| 398 |
+
@dataclass
|
| 399 |
+
class WeightTransformHooks:
|
| 400 |
+
split_hook: Optional[callable] = None
|
| 401 |
+
concat_hook: Optional[callable] = None
|
| 402 |
+
|
| 403 |
+
|
| 404 |
+
class TensorWeightAdapter:
|
| 405 |
+
"""
|
| 406 |
+
A wrapper for weight conversion hooks (split and concat) to be used with MemoryEfficientSafeOpen.
|
| 407 |
+
This wrapper adapts the original MemoryEfficientSafeOpen to apply the provided split and concat hooks
|
| 408 |
+
when loading tensors.
|
| 409 |
+
|
| 410 |
+
split_hook: A callable that takes (original_key: str, original_tensor: torch.Tensor) and returns (new_keys: list[str], new_tensors: list[torch.Tensor]).
|
| 411 |
+
concat_hook: A callable that takes (original_key: str, tensors: dict[str, torch.Tensor]) and returns (new_key: str, concatenated_tensor: torch.Tensor).
|
| 412 |
+
|
| 413 |
+
If tensors is None, the hook should return only the new keys (for split) or new key (for concat), without tensors.
|
| 414 |
+
|
| 415 |
+
No need to implement __enter__ and __exit__ methods, as they are handled by the original MemoryEfficientSafeOpen.
|
| 416 |
+
Do not use this wrapper as a context manager directly, like `with WeightConvertHookWrapper(...) as f:`.
|
| 417 |
+
|
| 418 |
+
**concat_hook is not tested yet.**
|
| 419 |
+
"""
|
| 420 |
+
|
| 421 |
+
def __init__(self, weight_convert_hook: WeightTransformHooks, original_f: MemoryEfficientSafeOpen):
|
| 422 |
+
self.original_f = original_f
|
| 423 |
+
self.new_key_to_original_key_map: dict[
|
| 424 |
+
str, Union[str, list[str]]
|
| 425 |
+
] = {} # for split: new_key -> original_key; for concat: new_key -> list of original_keys
|
| 426 |
+
self.concat_key_set = set() # set of keys that are created by concat_hook, to distinguish from split_hook
|
| 427 |
+
self.new_keys = []
|
| 428 |
+
self.tensor_cache = {} # cache for split tensors
|
| 429 |
+
self.split_hook = weight_convert_hook.split_hook
|
| 430 |
+
self.concat_hook = weight_convert_hook.concat_hook
|
| 431 |
+
|
| 432 |
+
for key in self.original_f.keys():
|
| 433 |
+
if self.split_hook is not None:
|
| 434 |
+
converted_keys, _ = self.split_hook(key, None) # get new keys only
|
| 435 |
+
if converted_keys is not None:
|
| 436 |
+
for new_key in converted_keys:
|
| 437 |
+
self.new_key_to_original_key_map[new_key] = key
|
| 438 |
+
self.new_keys.extend(converted_keys)
|
| 439 |
+
continue # skip concat_hook if split_hook is applied
|
| 440 |
+
|
| 441 |
+
if self.concat_hook is not None:
|
| 442 |
+
converted_key, _ = self.concat_hook(key, None) # get new key only
|
| 443 |
+
if converted_key is not None:
|
| 444 |
+
if converted_key not in self.concat_key_set: # first time seeing this concatenated key
|
| 445 |
+
self.concat_key_set.add(converted_key)
|
| 446 |
+
self.new_key_to_original_key_map[converted_key] = []
|
| 447 |
+
|
| 448 |
+
# multiple original keys map to the same concatenated key
|
| 449 |
+
self.new_key_to_original_key_map[converted_key].append(key)
|
| 450 |
+
|
| 451 |
+
self.new_keys.append(converted_key)
|
| 452 |
+
continue # skip to next key
|
| 453 |
+
|
| 454 |
+
# direct mapping
|
| 455 |
+
self.new_keys.append(key)
|
| 456 |
+
|
| 457 |
+
def keys(self) -> list[str]:
|
| 458 |
+
return self.new_keys
|
| 459 |
+
|
| 460 |
+
def get_tensor(self, new_key: str, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None) -> torch.Tensor:
|
| 461 |
+
# load tensor by new_key, applying split or concat hooks as needed
|
| 462 |
+
if new_key not in self.new_key_to_original_key_map:
|
| 463 |
+
# direct mapping
|
| 464 |
+
return self.original_f.get_tensor(new_key, device=device, dtype=dtype)
|
| 465 |
+
|
| 466 |
+
elif new_key not in self.concat_key_set:
|
| 467 |
+
# split hook: split key is requested multiple times, so we cache the result
|
| 468 |
+
original_key = self.new_key_to_original_key_map[new_key]
|
| 469 |
+
if original_key not in self.tensor_cache: # not yet split
|
| 470 |
+
original_tensor = self.original_f.get_tensor(original_key, device=device, dtype=dtype)
|
| 471 |
+
new_keys, new_tensors = self.split_hook(original_key, original_tensor) # apply split hook
|
| 472 |
+
for k, t in zip(new_keys, new_tensors):
|
| 473 |
+
self.tensor_cache[k] = t
|
| 474 |
+
return self.tensor_cache.pop(new_key) # return and remove from cache
|
| 475 |
+
|
| 476 |
+
else:
|
| 477 |
+
# concat hook: concatenated key is requested only once, so we do not cache the result
|
| 478 |
+
tensors = {}
|
| 479 |
+
for original_key in self.new_key_to_original_key_map[new_key]:
|
| 480 |
+
tensor = self.original_f.get_tensor(original_key, device=device, dtype=dtype)
|
| 481 |
+
tensors[original_key] = tensor
|
| 482 |
+
_, concatenated_tensors = self.concat_hook(self.new_key_to_original_key_map[new_key][0], tensors) # apply concat hook
|
| 483 |
+
return concatenated_tensors
|