Text-to-Video
Diffusers
Safetensors
English
orbitquant
w4a8
4-bit precision
gptq
quantized
vimoo
image-to-video
Instructions to use WaveCut/Vimoo-1.0-OrbitQuant-W4A8 with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Diffusers
How to use WaveCut/Vimoo-1.0-OrbitQuant-W4A8 with Diffusers:
pip install -U diffusers transformers accelerate
import torch from diffusers import DiffusionPipeline # switch to "mps" for apple devices pipe = DiffusionPipeline.from_pretrained("WaveCut/Vimoo-1.0-OrbitQuant-W4A8", dtype=torch.bfloat16, device_map="cuda") prompt = "Astronaut in a jungle, cold color palette, muted colors, detailed, 8k" image = pipe(prompt).images[0] - Notebooks
- Google Colab
- Kaggle
Download vimoo_orbitquant.py from WaveCut/Vimoo-1.0-OrbitQuant-W4A8: direct link, hf CLI and curl.
- Browser
- Download file 11.7 kB
-
https://huggingface.co/WaveCut/Vimoo-1.0-OrbitQuant-W4A8/resolve/main/vimoo_orbitquant.py
- Command line
-
hf download hf://WaveCut/Vimoo-1.0-OrbitQuant-W4A8/vimoo_orbitquant.py
-
curl -L -o vimoo_orbitquant.py https://huggingface.co/WaveCut/Vimoo-1.0-OrbitQuant-W4A8/resolve/main/vimoo_orbitquant.py
11.7 kB
| """Load a Vimoo OrbitQuant W4A8 bundle into the official Vimoo modular pipeline. | |
| The bundle replaces two components of the original model: the transformer (``transformer/``, and ``transformer_i2v/`` | |
| for Vimoo-1.0) and the Qwen2.5-VL text tower (``text_encoder/``). Both run 8-bit activations over 4-bit weights with | |
| the fused OrbitQuant kernels. Every other component is the original one: the ByT5 glyph encoder ships in the bundle, | |
| and the Wan2.2 autoencoder and the SigLIP image encoder download from their upstream repositories, as with the | |
| original model. | |
| from vimoo_orbitquant import load_pipeline | |
| pipe = load_pipeline(".", task="t2v") | |
| videos = pipe(prompt="...", **pipe.vimoo_defaults) | |
| """ | |
| from __future__ import annotations | |
| import dataclasses | |
| import json | |
| from pathlib import Path | |
| import torch | |
| NEGATIVE_LITE = ( | |
| "oversaturated colors, overexposed, static, blurry details, subtitles, style, artwork, painting, still image, " | |
| "grayish overall tone, worst quality, low quality, JPEG artifacts, ugly, mutilated, extra fingers, poorly drawn " | |
| "hands, poorly drawn face, deformed, disfigured, malformed limbs, fused fingers, motionless scene, cluttered " | |
| "background, three legs, many people in the background, walking backwards" | |
| ) | |
| # The official examples of each model (Vimoo-1.0 examples/*.py). | |
| PRESETS = { | |
| ("standard", "t2v"): dict( | |
| dtypes={"vision_encoder": torch.bfloat16, "default": torch.float32}, | |
| text_dtype=torch.bfloat16, guider=dict(guidance_scale=4.5, guidance_rescale=1.2), | |
| call=dict(negative_prompt=None, flow_shift=12.0), | |
| ), | |
| ("standard", "i2v"): dict( | |
| dtypes={"vision_encoder": torch.bfloat16, "default": torch.float32}, | |
| cast={"vision_encoder": torch.float16}, | |
| text_dtype=torch.float16, guider=dict(guidance_scale=3.0, guidance_rescale=0.5), scheduler_shift=6.0, | |
| call=dict(negative_prompt=None, add_fps=True, i2v_first_frame_replace=True, flow_shift=6.0), | |
| ), | |
| ("lite", "t2v"): dict( | |
| dtypes={"vision_encoder": torch.bfloat16, "default": torch.float32}, | |
| text_dtype=torch.float16, guider=dict(guidance_scale=4.0, guidance_rescale=0.85), | |
| call=dict(negative_prompt=NEGATIVE_LITE, flow_shift=8.0, max_sequence_length=1400), glyph_cpu=True, | |
| ), | |
| ("lite", "i2v"): dict( | |
| dtypes={"vision_encoder": torch.bfloat16, "default": torch.float32}, | |
| cast={"vision_encoder": torch.float16}, | |
| text_dtype=torch.float16, guider=dict(guidance_scale=5.0, guidance_rescale=0.85), | |
| call=dict(negative_prompt=NEGATIVE_LITE, flow_shift=8.0, max_sequence_length=1400, | |
| reference_guidance_scale=3.0, i2v_anchor_tau=0.001), | |
| glyph_cpu=True, | |
| ), | |
| } | |
| COMMON = dict(height=720, width=1280, num_frames=121, fps=24, num_inference_steps=50, vae_tiling=False) | |
| def load_transformer(path, *, attention="cudnn", device="cuda"): | |
| """The packed W4A8 transformer in the diffusers module layout, with fused blocks.""" | |
| import orbitquant # noqa: F401 (registers the OrbitQuant layers) | |
| from diffusers import VimooTransformer3DModel | |
| from orbitquant.artifacts.manifest import OrbitQuantManifest | |
| from orbitquant.config import OrbitQuantConfig | |
| from orbitquant.fused import fuse | |
| from orbitquant.layers import OrbitQuantLinear | |
| from orbitquant.modeling import _parent_and_child, _set_child | |
| from safetensors.torch import load_file | |
| path = Path(path) | |
| config = OrbitQuantConfig.from_dict(json.loads((path / "quantization_config.json").read_text())) | |
| manifest = OrbitQuantManifest.from_dict(json.loads((path / "orbitquant_manifest.json").read_text())) | |
| with torch.device("meta"): | |
| model = VimooTransformer3DModel.from_config(VimooTransformer3DModel.load_config(str(path))) | |
| for name in manifest.quantized_modules: | |
| bits = manifest.module_bits.get(name) | |
| layer_config = dataclasses.replace(config, weight_bits=bits) if bits and bits != config.weight_bits else config | |
| parent, child = _parent_and_child(model, name) | |
| _set_child(parent, child, OrbitQuantLinear.empty_from_linear(model.get_submodule(name), config=layer_config, | |
| module_name=name)) | |
| model.load_state_dict(load_file(str(path / "model.safetensors"), device=str(device)), strict=True, assign=True) | |
| model = model.to(device).eval() | |
| fuse(model, config=config, attention=attention) | |
| _chunk_final_layer(model.final_layer) | |
| return model | |
| def _chunk_final_layer(layer, tokens=8192): | |
| """Run the output layer over chunks of tokens. Every operation in it is per token, but at 1280x720x121 it builds | |
| FP32 copies of the whole batch of tokens, about 4 GB each with three guidance branches, which decided the memory | |
| peak on 32 GB cards.""" | |
| forward = layer.forward | |
| def chunked(x, c): | |
| if x.shape[1] <= tokens: | |
| return forward(x, c) | |
| return torch.cat([forward(x[:, i : i + tokens], c) for i in range(0, x.shape[1], tokens)], dim=1) | |
| layer.forward = chunked | |
| def _skip_layer(layer): | |
| """Layers 26-27 of the text tower: the pipeline reads hidden_states[-3], so their output never reaches the | |
| transformer. Each becomes a pass-through without weights. The module stays in place because transformers records | |
| hidden_states with hooks on the decoder layers; replacing it would shift hidden_states[-3].""" | |
| import types | |
| for name in list(layer._modules): | |
| setattr(layer, name, None) | |
| layer.forward = types.MethodType(lambda self, hidden_states, *args, **kwargs: hidden_states, layer) | |
| class TextEncoder(torch.nn.Module): | |
| """The W4A8 text tower (BF16 compute) with the dtype that the pipeline profile expects for its hidden states.""" | |
| def __init__(self, inner, dtype): | |
| super().__init__() | |
| self.inner = inner | |
| self._dtype = dtype | |
| def dtype(self): | |
| return self._dtype | |
| def config(self): | |
| return self.inner.config | |
| def device(self): | |
| return self.inner.device | |
| def forward(self, *args, **kwargs): | |
| out = self.inner(*args, **kwargs) | |
| if getattr(out, "hidden_states", None) is not None: | |
| out.hidden_states = tuple(h.to(self._dtype) for h in out.hidden_states) | |
| if getattr(out, "last_hidden_state", None) is not None: | |
| out.last_hidden_state = out.last_hidden_state.to(self._dtype) | |
| return out | |
| def load_text_encoder(path, *, dtype=torch.bfloat16, device="cuda"): | |
| """``dtype`` is the precision of the original encoder in the official pipeline (BF16 for Vimoo-1.0 T2V, FP16 | |
| otherwise). The quantized encoder computes in BF16; its rotary frequencies are rounded to ``dtype`` as the | |
| original's are, and its hidden states come back in ``dtype``.""" | |
| import orbitquant # noqa: F401 | |
| from orbitquant.fused import fuse_linears | |
| from transformers import Qwen2_5_VLTextModel | |
| model = Qwen2_5_VLTextModel.from_pretrained(str(path), dtype=torch.bfloat16, device_map={"": device}).eval() | |
| rotary = model.rotary_emb | |
| for name, buffer in list(rotary.named_buffers(recurse=False)): | |
| if buffer.is_floating_point(): | |
| setattr(rotary, name, buffer.to(dtype)) | |
| if torch.is_tensor(getattr(rotary, "original_inv_freq", None)): | |
| rotary.original_inv_freq = rotary.original_inv_freq.to(dtype) | |
| fuse_linears(model, activation_bits=8) | |
| model.config.use_cache = False | |
| for layer in list(model.layers)[26:]: | |
| _skip_layer(layer) | |
| model.norm = torch.nn.Identity() | |
| return model if dtype == torch.bfloat16 else TextEncoder(model, dtype) | |
| def load_pipeline(root=".", *, task="t2v", offload=False, attention="cudnn", vae_tiling=None, device="cuda"): | |
| """The official ``ModularPipeline`` of the bundle with the W4A8 transformer and text encoder. | |
| ``task``: "t2v" or "i2v". ``offload``: keep every model in host memory and move each to the GPU for its own stage | |
| (the official examples' auto CPU offload). ``attention``: "cudnn" (default), "flash", "sdpa" or "int8" (INT8 | |
| Q·K attention: faster, changes the video slightly). | |
| ``vae_tiling``: decode the video in tiles. ``None`` (default) tiles on GPUs with less than 60 GiB of memory: | |
| a 1280x720, 121-frame decode does not fit next to the models on a 32 GB card otherwise. The official examples | |
| decode without tiles. | |
| ``pipe.vimoo_defaults`` holds the official call arguments for the task; pass them with the prompt. | |
| """ | |
| from accelerate.hooks import remove_hook_from_module | |
| from diffusers import ClassifierFreeGuidance, ComponentsManager, ModularPipeline | |
| root = Path(root) | |
| manifest = json.loads((root / "manifest.json").read_text()) | |
| if manifest.get("format") != "vimoo-orbitquant": | |
| raise ValueError(f"{root} is not a Vimoo OrbitQuant release") | |
| kind = manifest["numerical_profile"] | |
| preset = PRESETS[(kind, task)] | |
| manager = ComponentsManager() | |
| pipe = ModularPipeline.from_pretrained(str(root), workflow=task, components_manager=manager) | |
| replaced = {"transformer", "transformer_i2v", "text_encoder"} | |
| needed = {spec.name for spec in pipe.blocks.get_workflow(task).expected_components} | |
| names = [n for n in pipe.pretrained_component_names if n in needed and n not in replaced] | |
| pipe.load_components(names=names, dtype=preset["dtypes"]) | |
| for name, dtype in preset.get("cast", {}).items(): | |
| component = pipe.components.get(name) | |
| if component is not None: | |
| component.to(dtype=dtype) | |
| sub = "transformer_i2v" if task == "i2v" and (root / "transformer_i2v").is_dir() else "transformer" | |
| updates = { | |
| "transformer": load_transformer(root / sub, attention=attention, device=device), | |
| "text_encoder": load_text_encoder(root / "text_encoder", dtype=preset["text_dtype"], device=device), | |
| "guider": ClassifierFreeGuidance.from_config(pipe.guider.config, **preset["guider"]), | |
| } | |
| if "scheduler_shift" in preset: | |
| from diffusers import VimooI2VFlowMatchEulerDiscreteScheduler | |
| updates["scheduler"] = VimooI2VFlowMatchEulerDiscreteScheduler(num_train_timesteps=1000, | |
| shift=preset["scheduler_shift"]) | |
| pipe.update_components(**updates) | |
| torch.cuda.empty_cache() | |
| if offload: | |
| from orbitquant.fused import release | |
| for name, component in pipe.components.items(): | |
| if isinstance(component, torch.nn.Module): | |
| if name in ("transformer", "text_encoder"): | |
| release(component) # drop the fused runtime's GPU work buffers before the move | |
| component.to("cpu") | |
| torch.cuda.empty_cache() | |
| manager.enable_auto_cpu_offload(device=torch.device(device), offload_strategy=lambda hooks, **kw: hooks) | |
| else: | |
| for name, component in pipe.components.items(): | |
| if isinstance(component, torch.nn.Module) and not (preset.get("glyph_cpu") and name == "text_encoder_2"): | |
| component.to(device) | |
| if preset.get("glyph_cpu") and pipe.components.get("text_encoder_2") is not None: | |
| # Glyph conditioning runs on the CPU in FP32, as in the official Lite example. | |
| remove_hook_from_module(pipe.text_encoder_2, recurse=True) | |
| pipe.text_encoder_2.to(device="cpu", dtype=torch.float32) | |
| if vae_tiling is None: | |
| vae_tiling = torch.cuda.get_device_properties(torch.device(device)).total_memory < 60 * 2**30 | |
| pipe.__dict__["vimoo_defaults"] = {**COMMON, **preset["call"], "vae_tiling": bool(vae_tiling)} | |
| pipe.__dict__["_orbitquant_manager"] = manager | |
| return pipe | |