Vimoo-1.0-OrbitQuant-W4A8 / vimoo_orbitquant.py
WaveCut's picture
Runtime files for orbitquant 0.17.1, card videos; drop the generic artifact README and prompt pack
fd9e7d5 verified
Raw History Blame Contribute Delete
11.7 kB
"""Load a Vimoo OrbitQuant W4A8 bundle into the official Vimoo modular pipeline.
The bundle replaces two components of the original model: the transformer (``transformer/``, and ``transformer_i2v/``
for Vimoo-1.0) and the Qwen2.5-VL text tower (``text_encoder/``). Both run 8-bit activations over 4-bit weights with
the fused OrbitQuant kernels. Every other component is the original one: the ByT5 glyph encoder ships in the bundle,
and the Wan2.2 autoencoder and the SigLIP image encoder download from their upstream repositories, as with the
original model.
from vimoo_orbitquant import load_pipeline
pipe = load_pipeline(".", task="t2v")
videos = pipe(prompt="...", **pipe.vimoo_defaults)
"""
from __future__ import annotations
import dataclasses
import json
from pathlib import Path
import torch
NEGATIVE_LITE = (
"oversaturated colors, overexposed, static, blurry details, subtitles, style, artwork, painting, still image, "
"grayish overall tone, worst quality, low quality, JPEG artifacts, ugly, mutilated, extra fingers, poorly drawn "
"hands, poorly drawn face, deformed, disfigured, malformed limbs, fused fingers, motionless scene, cluttered "
"background, three legs, many people in the background, walking backwards"
)
# The official examples of each model (Vimoo-1.0 examples/*.py).
PRESETS = {
("standard", "t2v"): dict(
dtypes={"vision_encoder": torch.bfloat16, "default": torch.float32},
text_dtype=torch.bfloat16, guider=dict(guidance_scale=4.5, guidance_rescale=1.2),
call=dict(negative_prompt=None, flow_shift=12.0),
),
("standard", "i2v"): dict(
dtypes={"vision_encoder": torch.bfloat16, "default": torch.float32},
cast={"vision_encoder": torch.float16},
text_dtype=torch.float16, guider=dict(guidance_scale=3.0, guidance_rescale=0.5), scheduler_shift=6.0,
call=dict(negative_prompt=None, add_fps=True, i2v_first_frame_replace=True, flow_shift=6.0),
),
("lite", "t2v"): dict(
dtypes={"vision_encoder": torch.bfloat16, "default": torch.float32},
text_dtype=torch.float16, guider=dict(guidance_scale=4.0, guidance_rescale=0.85),
call=dict(negative_prompt=NEGATIVE_LITE, flow_shift=8.0, max_sequence_length=1400), glyph_cpu=True,
),
("lite", "i2v"): dict(
dtypes={"vision_encoder": torch.bfloat16, "default": torch.float32},
cast={"vision_encoder": torch.float16},
text_dtype=torch.float16, guider=dict(guidance_scale=5.0, guidance_rescale=0.85),
call=dict(negative_prompt=NEGATIVE_LITE, flow_shift=8.0, max_sequence_length=1400,
reference_guidance_scale=3.0, i2v_anchor_tau=0.001),
glyph_cpu=True,
),
}
COMMON = dict(height=720, width=1280, num_frames=121, fps=24, num_inference_steps=50, vae_tiling=False)
def load_transformer(path, *, attention="cudnn", device="cuda"):
"""The packed W4A8 transformer in the diffusers module layout, with fused blocks."""
import orbitquant # noqa: F401 (registers the OrbitQuant layers)
from diffusers import VimooTransformer3DModel
from orbitquant.artifacts.manifest import OrbitQuantManifest
from orbitquant.config import OrbitQuantConfig
from orbitquant.fused import fuse
from orbitquant.layers import OrbitQuantLinear
from orbitquant.modeling import _parent_and_child, _set_child
from safetensors.torch import load_file
path = Path(path)
config = OrbitQuantConfig.from_dict(json.loads((path / "quantization_config.json").read_text()))
manifest = OrbitQuantManifest.from_dict(json.loads((path / "orbitquant_manifest.json").read_text()))
with torch.device("meta"):
model = VimooTransformer3DModel.from_config(VimooTransformer3DModel.load_config(str(path)))
for name in manifest.quantized_modules:
bits = manifest.module_bits.get(name)
layer_config = dataclasses.replace(config, weight_bits=bits) if bits and bits != config.weight_bits else config
parent, child = _parent_and_child(model, name)
_set_child(parent, child, OrbitQuantLinear.empty_from_linear(model.get_submodule(name), config=layer_config,
module_name=name))
model.load_state_dict(load_file(str(path / "model.safetensors"), device=str(device)), strict=True, assign=True)
model = model.to(device).eval()
fuse(model, config=config, attention=attention)
_chunk_final_layer(model.final_layer)
return model
def _chunk_final_layer(layer, tokens=8192):
"""Run the output layer over chunks of tokens. Every operation in it is per token, but at 1280x720x121 it builds
FP32 copies of the whole batch of tokens, about 4 GB each with three guidance branches, which decided the memory
peak on 32 GB cards."""
forward = layer.forward
def chunked(x, c):
if x.shape[1] <= tokens:
return forward(x, c)
return torch.cat([forward(x[:, i : i + tokens], c) for i in range(0, x.shape[1], tokens)], dim=1)
layer.forward = chunked
def _skip_layer(layer):
"""Layers 26-27 of the text tower: the pipeline reads hidden_states[-3], so their output never reaches the
transformer. Each becomes a pass-through without weights. The module stays in place because transformers records
hidden_states with hooks on the decoder layers; replacing it would shift hidden_states[-3]."""
import types
for name in list(layer._modules):
setattr(layer, name, None)
layer.forward = types.MethodType(lambda self, hidden_states, *args, **kwargs: hidden_states, layer)
class TextEncoder(torch.nn.Module):
"""The W4A8 text tower (BF16 compute) with the dtype that the pipeline profile expects for its hidden states."""
def __init__(self, inner, dtype):
super().__init__()
self.inner = inner
self._dtype = dtype
@property
def dtype(self):
return self._dtype
@property
def config(self):
return self.inner.config
@property
def device(self):
return self.inner.device
def forward(self, *args, **kwargs):
out = self.inner(*args, **kwargs)
if getattr(out, "hidden_states", None) is not None:
out.hidden_states = tuple(h.to(self._dtype) for h in out.hidden_states)
if getattr(out, "last_hidden_state", None) is not None:
out.last_hidden_state = out.last_hidden_state.to(self._dtype)
return out
def load_text_encoder(path, *, dtype=torch.bfloat16, device="cuda"):
"""``dtype`` is the precision of the original encoder in the official pipeline (BF16 for Vimoo-1.0 T2V, FP16
otherwise). The quantized encoder computes in BF16; its rotary frequencies are rounded to ``dtype`` as the
original's are, and its hidden states come back in ``dtype``."""
import orbitquant # noqa: F401
from orbitquant.fused import fuse_linears
from transformers import Qwen2_5_VLTextModel
model = Qwen2_5_VLTextModel.from_pretrained(str(path), dtype=torch.bfloat16, device_map={"": device}).eval()
rotary = model.rotary_emb
for name, buffer in list(rotary.named_buffers(recurse=False)):
if buffer.is_floating_point():
setattr(rotary, name, buffer.to(dtype))
if torch.is_tensor(getattr(rotary, "original_inv_freq", None)):
rotary.original_inv_freq = rotary.original_inv_freq.to(dtype)
fuse_linears(model, activation_bits=8)
model.config.use_cache = False
for layer in list(model.layers)[26:]:
_skip_layer(layer)
model.norm = torch.nn.Identity()
return model if dtype == torch.bfloat16 else TextEncoder(model, dtype)
def load_pipeline(root=".", *, task="t2v", offload=False, attention="cudnn", vae_tiling=None, device="cuda"):
"""The official ``ModularPipeline`` of the bundle with the W4A8 transformer and text encoder.
``task``: "t2v" or "i2v". ``offload``: keep every model in host memory and move each to the GPU for its own stage
(the official examples' auto CPU offload). ``attention``: "cudnn" (default), "flash", "sdpa" or "int8" (INT8
Q·K attention: faster, changes the video slightly).
``vae_tiling``: decode the video in tiles. ``None`` (default) tiles on GPUs with less than 60 GiB of memory:
a 1280x720, 121-frame decode does not fit next to the models on a 32 GB card otherwise. The official examples
decode without tiles.
``pipe.vimoo_defaults`` holds the official call arguments for the task; pass them with the prompt.
"""
from accelerate.hooks import remove_hook_from_module
from diffusers import ClassifierFreeGuidance, ComponentsManager, ModularPipeline
root = Path(root)
manifest = json.loads((root / "manifest.json").read_text())
if manifest.get("format") != "vimoo-orbitquant":
raise ValueError(f"{root} is not a Vimoo OrbitQuant release")
kind = manifest["numerical_profile"]
preset = PRESETS[(kind, task)]
manager = ComponentsManager()
pipe = ModularPipeline.from_pretrained(str(root), workflow=task, components_manager=manager)
replaced = {"transformer", "transformer_i2v", "text_encoder"}
needed = {spec.name for spec in pipe.blocks.get_workflow(task).expected_components}
names = [n for n in pipe.pretrained_component_names if n in needed and n not in replaced]
pipe.load_components(names=names, dtype=preset["dtypes"])
for name, dtype in preset.get("cast", {}).items():
component = pipe.components.get(name)
if component is not None:
component.to(dtype=dtype)
sub = "transformer_i2v" if task == "i2v" and (root / "transformer_i2v").is_dir() else "transformer"
updates = {
"transformer": load_transformer(root / sub, attention=attention, device=device),
"text_encoder": load_text_encoder(root / "text_encoder", dtype=preset["text_dtype"], device=device),
"guider": ClassifierFreeGuidance.from_config(pipe.guider.config, **preset["guider"]),
}
if "scheduler_shift" in preset:
from diffusers import VimooI2VFlowMatchEulerDiscreteScheduler
updates["scheduler"] = VimooI2VFlowMatchEulerDiscreteScheduler(num_train_timesteps=1000,
shift=preset["scheduler_shift"])
pipe.update_components(**updates)
torch.cuda.empty_cache()
if offload:
from orbitquant.fused import release
for name, component in pipe.components.items():
if isinstance(component, torch.nn.Module):
if name in ("transformer", "text_encoder"):
release(component) # drop the fused runtime's GPU work buffers before the move
component.to("cpu")
torch.cuda.empty_cache()
manager.enable_auto_cpu_offload(device=torch.device(device), offload_strategy=lambda hooks, **kw: hooks)
else:
for name, component in pipe.components.items():
if isinstance(component, torch.nn.Module) and not (preset.get("glyph_cpu") and name == "text_encoder_2"):
component.to(device)
if preset.get("glyph_cpu") and pipe.components.get("text_encoder_2") is not None:
# Glyph conditioning runs on the CPU in FP32, as in the official Lite example.
remove_hook_from_module(pipe.text_encoder_2, recurse=True)
pipe.text_encoder_2.to(device="cpu", dtype=torch.float32)
if vae_tiling is None:
vae_tiling = torch.cuda.get_device_properties(torch.device(device)).total_memory < 60 * 2**30
pipe.__dict__["vimoo_defaults"] = {**COMMON, **preset["call"], "vae_tiling": bool(vae_tiling)}
pipe.__dict__["_orbitquant_manager"] = manager
return pipe