"""Load a Vimoo OrbitQuant W4A8 bundle into the official Vimoo modular pipeline. The bundle replaces two components of the original model: the transformer (``transformer/``, and ``transformer_i2v/`` for Vimoo-1.0) and the Qwen2.5-VL text tower (``text_encoder/``). Both run 8-bit activations over 4-bit weights with the fused OrbitQuant kernels. Every other component is the original one: the ByT5 glyph encoder ships in the bundle, and the Wan2.2 autoencoder and the SigLIP image encoder download from their upstream repositories, as with the original model. from vimoo_orbitquant import load_pipeline pipe = load_pipeline(".", task="t2v") videos = pipe(prompt="...", **pipe.vimoo_defaults) """ from __future__ import annotations import dataclasses import json from pathlib import Path import torch NEGATIVE_LITE = ( "oversaturated colors, overexposed, static, blurry details, subtitles, style, artwork, painting, still image, " "grayish overall tone, worst quality, low quality, JPEG artifacts, ugly, mutilated, extra fingers, poorly drawn " "hands, poorly drawn face, deformed, disfigured, malformed limbs, fused fingers, motionless scene, cluttered " "background, three legs, many people in the background, walking backwards" ) # The official examples of each model (Vimoo-1.0 examples/*.py). PRESETS = { ("standard", "t2v"): dict( dtypes={"vision_encoder": torch.bfloat16, "default": torch.float32}, text_dtype=torch.bfloat16, guider=dict(guidance_scale=4.5, guidance_rescale=1.2), call=dict(negative_prompt=None, flow_shift=12.0), ), ("standard", "i2v"): dict( dtypes={"vision_encoder": torch.bfloat16, "default": torch.float32}, cast={"vision_encoder": torch.float16}, text_dtype=torch.float16, guider=dict(guidance_scale=3.0, guidance_rescale=0.5), scheduler_shift=6.0, call=dict(negative_prompt=None, add_fps=True, i2v_first_frame_replace=True, flow_shift=6.0), ), ("lite", "t2v"): dict( dtypes={"vision_encoder": torch.bfloat16, "default": torch.float32}, text_dtype=torch.float16, guider=dict(guidance_scale=4.0, guidance_rescale=0.85), call=dict(negative_prompt=NEGATIVE_LITE, flow_shift=8.0, max_sequence_length=1400), glyph_cpu=True, ), ("lite", "i2v"): dict( dtypes={"vision_encoder": torch.bfloat16, "default": torch.float32}, cast={"vision_encoder": torch.float16}, text_dtype=torch.float16, guider=dict(guidance_scale=5.0, guidance_rescale=0.85), call=dict(negative_prompt=NEGATIVE_LITE, flow_shift=8.0, max_sequence_length=1400, reference_guidance_scale=3.0, i2v_anchor_tau=0.001), glyph_cpu=True, ), } COMMON = dict(height=720, width=1280, num_frames=121, fps=24, num_inference_steps=50, vae_tiling=False) def load_transformer(path, *, attention="cudnn", device="cuda"): """The packed W4A8 transformer in the diffusers module layout, with fused blocks.""" import orbitquant # noqa: F401 (registers the OrbitQuant layers) from diffusers import VimooTransformer3DModel from orbitquant.artifacts.manifest import OrbitQuantManifest from orbitquant.config import OrbitQuantConfig from orbitquant.fused import fuse from orbitquant.layers import OrbitQuantLinear from orbitquant.modeling import _parent_and_child, _set_child from safetensors.torch import load_file path = Path(path) config = OrbitQuantConfig.from_dict(json.loads((path / "quantization_config.json").read_text())) manifest = OrbitQuantManifest.from_dict(json.loads((path / "orbitquant_manifest.json").read_text())) with torch.device("meta"): model = VimooTransformer3DModel.from_config(VimooTransformer3DModel.load_config(str(path))) for name in manifest.quantized_modules: bits = manifest.module_bits.get(name) layer_config = dataclasses.replace(config, weight_bits=bits) if bits and bits != config.weight_bits else config parent, child = _parent_and_child(model, name) _set_child(parent, child, OrbitQuantLinear.empty_from_linear(model.get_submodule(name), config=layer_config, module_name=name)) model.load_state_dict(load_file(str(path / "model.safetensors"), device=str(device)), strict=True, assign=True) model = model.to(device).eval() fuse(model, config=config, attention=attention) _chunk_final_layer(model.final_layer) return model def _chunk_final_layer(layer, tokens=8192): """Run the output layer over chunks of tokens. Every operation in it is per token, but at 1280x720x121 it builds FP32 copies of the whole batch of tokens, about 4 GB each with three guidance branches, which decided the memory peak on 32 GB cards.""" forward = layer.forward def chunked(x, c): if x.shape[1] <= tokens: return forward(x, c) return torch.cat([forward(x[:, i : i + tokens], c) for i in range(0, x.shape[1], tokens)], dim=1) layer.forward = chunked def _skip_layer(layer): """Layers 26-27 of the text tower: the pipeline reads hidden_states[-3], so their output never reaches the transformer. Each becomes a pass-through without weights. The module stays in place because transformers records hidden_states with hooks on the decoder layers; replacing it would shift hidden_states[-3].""" import types for name in list(layer._modules): setattr(layer, name, None) layer.forward = types.MethodType(lambda self, hidden_states, *args, **kwargs: hidden_states, layer) class TextEncoder(torch.nn.Module): """The W4A8 text tower (BF16 compute) with the dtype that the pipeline profile expects for its hidden states.""" def __init__(self, inner, dtype): super().__init__() self.inner = inner self._dtype = dtype @property def dtype(self): return self._dtype @property def config(self): return self.inner.config @property def device(self): return self.inner.device def forward(self, *args, **kwargs): out = self.inner(*args, **kwargs) if getattr(out, "hidden_states", None) is not None: out.hidden_states = tuple(h.to(self._dtype) for h in out.hidden_states) if getattr(out, "last_hidden_state", None) is not None: out.last_hidden_state = out.last_hidden_state.to(self._dtype) return out def load_text_encoder(path, *, dtype=torch.bfloat16, device="cuda"): """``dtype`` is the precision of the original encoder in the official pipeline (BF16 for Vimoo-1.0 T2V, FP16 otherwise). The quantized encoder computes in BF16; its rotary frequencies are rounded to ``dtype`` as the original's are, and its hidden states come back in ``dtype``.""" import orbitquant # noqa: F401 from orbitquant.fused import fuse_linears from transformers import Qwen2_5_VLTextModel model = Qwen2_5_VLTextModel.from_pretrained(str(path), dtype=torch.bfloat16, device_map={"": device}).eval() rotary = model.rotary_emb for name, buffer in list(rotary.named_buffers(recurse=False)): if buffer.is_floating_point(): setattr(rotary, name, buffer.to(dtype)) if torch.is_tensor(getattr(rotary, "original_inv_freq", None)): rotary.original_inv_freq = rotary.original_inv_freq.to(dtype) fuse_linears(model, activation_bits=8) model.config.use_cache = False for layer in list(model.layers)[26:]: _skip_layer(layer) model.norm = torch.nn.Identity() return model if dtype == torch.bfloat16 else TextEncoder(model, dtype) def load_pipeline(root=".", *, task="t2v", offload=False, attention="cudnn", vae_tiling=None, device="cuda"): """The official ``ModularPipeline`` of the bundle with the W4A8 transformer and text encoder. ``task``: "t2v" or "i2v". ``offload``: keep every model in host memory and move each to the GPU for its own stage (the official examples' auto CPU offload). ``attention``: "cudnn" (default), "flash", "sdpa" or "int8" (INT8 Q·K attention: faster, changes the video slightly). ``vae_tiling``: decode the video in tiles. ``None`` (default) tiles on GPUs with less than 60 GiB of memory: a 1280x720, 121-frame decode does not fit next to the models on a 32 GB card otherwise. The official examples decode without tiles. ``pipe.vimoo_defaults`` holds the official call arguments for the task; pass them with the prompt. """ from accelerate.hooks import remove_hook_from_module from diffusers import ClassifierFreeGuidance, ComponentsManager, ModularPipeline root = Path(root) manifest = json.loads((root / "manifest.json").read_text()) if manifest.get("format") != "vimoo-orbitquant": raise ValueError(f"{root} is not a Vimoo OrbitQuant release") kind = manifest["numerical_profile"] preset = PRESETS[(kind, task)] manager = ComponentsManager() pipe = ModularPipeline.from_pretrained(str(root), workflow=task, components_manager=manager) replaced = {"transformer", "transformer_i2v", "text_encoder"} needed = {spec.name for spec in pipe.blocks.get_workflow(task).expected_components} names = [n for n in pipe.pretrained_component_names if n in needed and n not in replaced] pipe.load_components(names=names, dtype=preset["dtypes"]) for name, dtype in preset.get("cast", {}).items(): component = pipe.components.get(name) if component is not None: component.to(dtype=dtype) sub = "transformer_i2v" if task == "i2v" and (root / "transformer_i2v").is_dir() else "transformer" updates = { "transformer": load_transformer(root / sub, attention=attention, device=device), "text_encoder": load_text_encoder(root / "text_encoder", dtype=preset["text_dtype"], device=device), "guider": ClassifierFreeGuidance.from_config(pipe.guider.config, **preset["guider"]), } if "scheduler_shift" in preset: from diffusers import VimooI2VFlowMatchEulerDiscreteScheduler updates["scheduler"] = VimooI2VFlowMatchEulerDiscreteScheduler(num_train_timesteps=1000, shift=preset["scheduler_shift"]) pipe.update_components(**updates) torch.cuda.empty_cache() if offload: from orbitquant.fused import release for name, component in pipe.components.items(): if isinstance(component, torch.nn.Module): if name in ("transformer", "text_encoder"): release(component) # drop the fused runtime's GPU work buffers before the move component.to("cpu") torch.cuda.empty_cache() manager.enable_auto_cpu_offload(device=torch.device(device), offload_strategy=lambda hooks, **kw: hooks) else: for name, component in pipe.components.items(): if isinstance(component, torch.nn.Module) and not (preset.get("glyph_cpu") and name == "text_encoder_2"): component.to(device) if preset.get("glyph_cpu") and pipe.components.get("text_encoder_2") is not None: # Glyph conditioning runs on the CPU in FP32, as in the official Lite example. remove_hook_from_module(pipe.text_encoder_2, recurse=True) pipe.text_encoder_2.to(device="cpu", dtype=torch.float32) if vae_tiling is None: vae_tiling = torch.cuda.get_device_properties(torch.device(device)).total_memory < 60 * 2**30 pipe.__dict__["vimoo_defaults"] = {**COMMON, **preset["call"], "vae_tiling": bool(vae_tiling)} pipe.__dict__["_orbitquant_manager"] = manager return pipe