# wanq.py -- weight-only low-bit quantization for the Wan2.1 video DiT (diffusers WanTransformer3DModel), Bora part 2. # # Storage: every nn.Linear inside transformer.blocks becomes a QuantLinear holding # qweight uint8, the integer codes bit-packed (2, 3, 4 or 8 bits per weight) # scale fp16 [out, in/group] zero fp16 [out, in/group] w = (code - zero) * scale # Weights are dequantized on the fly, one layer at a time, so GPU memory holds the packed model plus one bf16 layer. # # Quantizers (all asymmetric, per output row and per group of `group` input columns): # rtn round-to-nearest with min/max range # hqq Half-Quadratic Quantization (Badri & Shaji 2023): calibration-free, optimizes the zero point with an lp<1 proximal solver # gptq GPTQ (Frantar et al. 2023): column-by-column rounding with error feedback through the inverse Hessian of the layer input # Fine-tuning: QA-LoRA (Xu et al. 2024) adapters on group-pooled inputs, plus a learnable per-group log-scale. Both merge exactly # into (scale, zero) afterwards, so the fine-tuned model has the same size as the PTQ model. import copy import json import math import time import torch import torch.nn as nn import torch.nn.functional as F # ----------------------------------------------------------------------------- bit packing def packed_len(n, bits): assert n % 8 == 0, f"in_features {n} must be a multiple of 8" return n * bits // 8 def pack_codes(codes, bits): """codes uint8 [..., n] with values < 2**bits -> uint8 [..., n * bits / 8].""" *lead, n = codes.shape if bits == 8: return codes.contiguous() if bits == 4: c = codes.reshape(*lead, n // 2, 2) return (c[..., 0] | (c[..., 1] << 4)).contiguous() if bits == 2: c = codes.reshape(*lead, n // 4, 4) return (c[..., 0] | (c[..., 1] << 2) | (c[..., 2] << 4) | (c[..., 3] << 6)).contiguous() if bits == 3: # 8 codes -> 24 bits -> 3 bytes c = codes.reshape(*lead, n // 8, 8).to(torch.int32) shifts = torch.arange(8, device=codes.device, dtype=torch.int32) * 3 word = (c << shifts).sum(-1) out = torch.stack([(word >> (8 * j)) & 0xFF for j in range(3)], dim=-1).to(torch.uint8) return out.reshape(*lead, n // 8 * 3).contiguous() raise ValueError(f"unsupported bits {bits}") def unpack_codes(packed, bits, n): *lead, _ = packed.shape if bits == 8: return packed if bits == 4: return torch.stack([packed & 0x0F, packed >> 4], dim=-1).reshape(*lead, n) if bits == 2: return torch.stack([(packed >> s) & 0x03 for s in (0, 2, 4, 6)], dim=-1).reshape(*lead, n) if bits == 3: b = packed.reshape(*lead, n // 8, 3).to(torch.int32) word = b[..., 0] | (b[..., 1] << 8) | (b[..., 2] << 16) shifts = torch.arange(8, device=packed.device, dtype=torch.int32) * 3 return ((word.unsqueeze(-1) >> shifts) & 0x07).to(torch.uint8).reshape(*lead, n) raise ValueError(f"unsupported bits {bits}") # ----------------------------------------------------------------------------- the quantized layer class QuantLinear(nn.Module): def __init__(self, in_features, out_features, bits, group, bias=True, dtype=torch.bfloat16, device=None): super().__init__() assert in_features % group == 0, f"in_features {in_features} not divisible by group {group}" self.in_features, self.out_features, self.bits, self.group = in_features, out_features, bits, group self.ngroups = in_features // group self.register_buffer("qweight", torch.zeros(out_features, packed_len(in_features, bits), dtype=torch.uint8, device=device)) self.register_buffer("scale", torch.zeros(out_features, self.ngroups, dtype=torch.float16, device=device)) self.register_buffer("zero", torch.zeros(out_features, self.ngroups, dtype=torch.float16, device=device)) self.register_buffer("bias", torch.zeros(out_features, dtype=dtype, device=device) if bias else None) self.compute_dtype = dtype self.lora_A = self.lora_B = self.log_scale = None self.lora_scaling = 0.0 self._cache = None @classmethod def from_codes(cls, linear, codes, scale, zero, bits, group): q = cls(linear.in_features, linear.out_features, bits, group, bias=linear.bias is not None, dtype=linear.weight.dtype, device=linear.weight.device) q.qweight.copy_(pack_codes(codes.to(torch.uint8), bits)) q.scale.copy_(scale.to(torch.float16)) q.zero.copy_(zero.to(torch.float16)) if linear.bias is not None: q.bias.copy_(linear.bias.detach()) return q def codes(self): return unpack_codes(self.qweight, self.bits, self.in_features) def effective_scale(self): if self.log_scale is None: return self.scale return self.scale.float() * self.log_scale.exp() def dequant(self, dtype=None): """Dense weight [out, in]. Differentiable w.r.t. log_scale when that is set.""" dtype = dtype or self.compute_dtype c = self.codes().view(self.out_features, self.ngroups, self.group) w = (c.float() - self.zero.float().unsqueeze(-1)) * self.effective_scale().float().unsqueeze(-1) # fp32 math, one layer at a time return w.reshape(self.out_features, self.in_features).to(dtype) def dense_with_adapters(self): """fp32 weight including the adapter delta (for checking merges): W + scaling * B A / group, constant within each group.""" w = self.dequant(torch.float32) if self.lora_A is not None: m = (self.lora_B.float() @ self.lora_A.float()) * self.lora_scaling / self.group w = w + m.repeat_interleave(self.group, dim=1) return w def cache_dense(self, on=True): """Keep the dense weight around (faster sweeps; uses bf16 memory). Off by default.""" self._cache = self.dequant() if on else None def forward(self, x): w = self._cache if self._cache is not None else self.dequant(x.dtype) y = F.linear(x, w, self.bias) if self.lora_A is not None: pooled = x.reshape(*x.shape[:-1], self.ngroups, self.group).float().mean(-1) # QA-LoRA: one input per group y = y + (F.linear(F.linear(pooled, self.lora_A), self.lora_B) * self.lora_scaling).to(y.dtype) return y # ---- fine-tuning hooks ---- def add_adapters(self, rank, alpha=None, train_scale=True): dev = self.qweight.device self.lora_A = nn.Parameter(torch.randn(rank, self.ngroups, device=dev) / math.sqrt(self.ngroups)) self.lora_B = nn.Parameter(torch.zeros(self.out_features, rank, device=dev)) self.lora_scaling = (alpha or rank) / rank if train_scale: self.log_scale = nn.Parameter(torch.zeros(self.out_features, self.ngroups, device=dev)) @torch.no_grad() def merge_adapters(self): """Fold log_scale into scale and the QA-LoRA delta into zero. Exact up to fp16 rounding of the new scale/zero.""" s = self.effective_scale().float() z = self.zero.float() if self.lora_A is not None: m = (self.lora_B.float() @ self.lora_A.float()) * self.lora_scaling # [out, ngroups] z = z - m / (self.group * s) self.scale.copy_(s.half()) self.zero.copy_(z.half()) self.lora_A = self.lora_B = self.log_scale = None self.lora_scaling = 0.0 def extra_repr(self): return f"in={self.in_features}, out={self.out_features}, bits={self.bits}, group={self.group}" # ----------------------------------------------------------------------------- quantizers (fp32 W [out, in] -> codes, scale, zero) def _minmax_params(wg, maxq): mn, mx = wg.amin(-1), wg.amax(-1) scale = ((mx - mn) / maxq).clamp(min=1e-8) return mn, scale def quant_rtn(w, bits, group): maxq = 2 ** bits - 1 out, inp = w.shape wg = w.view(out, inp // group, group) mn, scale = _minmax_params(wg, maxq) zero = torch.round(-mn / scale) s16, z16 = scale.half().float(), zero.half().float() codes = torch.clamp(torch.round(wg / s16.unsqueeze(-1) + z16.unsqueeze(-1)), 0, maxq) return codes.reshape(out, inp).to(torch.uint8), s16, z16 def quant_hqq(w, bits, group, iters=20, lp=0.7, beta=10.0, kappa=1.01): """HQQ: keep the min/max scale, optimize a float zero point with a half-quadratic lp-norm (p<1) solver. No data needed.""" maxq = 2 ** bits - 1 out, inp = w.shape wg = w.view(out, inp // group, group).float() mn, scale = _minmax_params(wg, maxq) inv = (1.0 / scale).unsqueeze(-1) z = (-mn / scale).unsqueeze(-1) def shrink(x, b): return torch.sign(x) * F.relu(x.abs() - (1.0 / b) * x.abs().clamp(min=1e-12).pow(lp - 1)) best_err, best_z = float("inf"), z for _ in range(iters): wq = torch.clamp(torch.round(wg * inv + z), 0, maxq) wr = (wq - z) / inv err = (wg - wr).abs().mean().item() if err < best_err: best_err, best_z = err, z else: break we = shrink(wg - wr, beta) z = (wq - (wg - we) * inv).mean(-1, keepdim=True) beta *= kappa s16 = scale.half().float() z16 = best_z.squeeze(-1).half().float() codes = torch.clamp(torch.round(wg / s16.unsqueeze(-1) + z16.unsqueeze(-1)), 0, maxq) return codes.reshape(out, inp).to(torch.uint8), s16, z16 def quant_gptq(w, H, bits, group, damp=0.01, blocksize=128): """GPTQ with groupwise min/max parameters computed on the error-updated weights. H: [in, in] = E[x x^T] of the layer input.""" maxq = 2 ** bits - 1 W = w.clone().float() out, cols = W.shape assert blocksize % group == 0 or group % blocksize == 0 H = H.clone().float() dead = torch.diag(H) == 0 H[dead, dead] = 1 W[:, dead] = 0 H.diagonal().add_(damp * torch.mean(torch.diag(H))) try: L = torch.linalg.cholesky(H) except RuntimeError: # rare: not positive definite -> more damping H.diagonal().add_(0.1 * torch.mean(torch.diag(H))) L = torch.linalg.cholesky(H) Hinv = torch.linalg.cholesky(torch.cholesky_inverse(L), upper=True) codes = torch.zeros(out, cols, dtype=torch.uint8, device=W.device) scales = torch.zeros(out, cols // group, device=W.device) zeros = torch.zeros(out, cols // group, device=W.device) s = z = None for i1 in range(0, cols, blocksize): i2 = min(i1 + blocksize, cols) W1 = W[:, i1:i2].clone() Err1 = torch.zeros_like(W1) Hinv1 = Hinv[i1:i2, i1:i2] for i in range(i2 - i1): col = i1 + i if col % group == 0: # new group: parameters from the current (error-updated) weights gw = W1[:, i:i + group] if i + group <= i2 - i1 else W[:, col:col + group] mn, mx = gw.amin(1), gw.amax(1) s = ((mx - mn) / maxq).clamp(min=1e-8).half().float() z = torch.round(-mn / s).half().float() scales[:, col // group], zeros[:, col // group] = s, z wcol = W1[:, i] q = torch.clamp(torch.round(wcol / s + z), 0, maxq) codes[:, col] = q.to(torch.uint8) deq = (q - z) * s d = Hinv1[i, i] e = (wcol - deq) / d W1[:, i:] -= e.unsqueeze(1) @ Hinv1[i, i:].unsqueeze(0) Err1[:, i] = e W[:, i2:] -= Err1 @ Hinv[i1:i2, i2:] return codes, scales, zeros # ----------------------------------------------------------------------------- model-level helpers def block_linears(model): """(name, module) for every nn.Linear inside transformer.blocks. Embedders, norms and the output head stay bf16.""" return [(n, m) for n, m in model.named_modules() if n.startswith("blocks.") and isinstance(m, nn.Linear)] def _set_module(model, name, new): parent, _, child = name.rpartition(".") setattr(model.get_submodule(parent), child, new) @torch.no_grad() def collect_hessians(model, batches, log=print): """Run the bf16 model on calibration batches; accumulate H = sum x x^T / n for the input of every block linear (fp32, on GPU). batches: iterable of dicts with hidden_states, timestep, encoder_hidden_states (already on the model device).""" H, n, hooks = {}, {}, [] for name, mod in block_linears(model): H[name] = torch.zeros(mod.in_features, mod.in_features, device=mod.weight.device, dtype=torch.float32) n[name] = 0 def hook(m, inp, out, name=name): x = inp[0].detach().reshape(-1, inp[0].shape[-1]).float() H[name].addmm_(x.T, x) n[name] += x.shape[0] hooks.append(mod.register_forward_hook(hook)) t0 = time.time() try: for i, b in enumerate(batches): model(hidden_states=b["hidden_states"], timestep=b["timestep"], encoder_hidden_states=b["encoder_hidden_states"], return_dict=False) finally: for h in hooks: h.remove() for k in H: H[k] /= max(1, n[k]) gb = sum(h.numel() * 4 for h in H.values()) / 1024 ** 3 log(f"hessians: {len(H)} layers, {i + 1} batches, {time.time() - t0:.0f}s, {gb:.1f} GB") return H @torch.no_grad() def quantize_model(base, method, bits, group, hessians=None, log=print, **kw): """Copy of `base` with every block linear replaced by a QuantLinear. `base` is not modified.""" model = copy.deepcopy(base) t0 = time.time() err_num = err_den = 0.0 for name, lin in block_linears(model): w = lin.weight.detach().float() if method == "rtn": codes, s, z = quant_rtn(w, bits, group) elif method == "hqq": codes, s, z = quant_hqq(w, bits, group, **kw) elif method == "gptq": assert hessians is not None, "gptq needs hessians (collect_hessians)" codes, s, z = quant_gptq(w, hessians[name], bits, group, **kw) else: raise ValueError(method) q = QuantLinear.from_codes(lin, codes, s, z, bits, group) err_num += (q.dequant(torch.float32) - w).pow(2).sum().item() err_den += w.pow(2).sum().item() _set_module(model, name, q) log(f"{method} W{bits} g{group}: {len(block_linears(base))} layers in {time.time() - t0:.0f}s | relative weight error {math.sqrt(err_num / err_den):.4f}") return model def quant_spec(model): return {n: [m.bits, m.group] for n, m in model.named_modules() if isinstance(m, QuantLinear)} def model_bytes(model): """Bytes of everything the model stores (parameters + persistent buffers), i.e. the checkpoint size.""" return sum(t.numel() * t.element_size() for t in model.state_dict().values()) def size_report(model): q = [m for m in model.modules() if isinstance(m, QuantLinear)] qw = sum(m.in_features * m.out_features for m in q) qbytes = sum(m.qweight.numel() + 2 * m.scale.numel() + 2 * m.zero.numel() for m in q) total = model_bytes(model) return {"total_MB": total / 1e6, "quantized_weights": qw, "quantized_MB": qbytes / 1e6, # decimal MB, like "Bora = 673 MB" "bits_per_quantized_weight": 8 * qbytes / max(1, qw), "other_MB": (total - qbytes) / 1e6} def save_quantized(model, path, extra=None): from safetensors.torch import save_file sd = {k: v.contiguous() for k, v in model.state_dict().items()} persistent = set(model.state_dict().keys()) buf_dtypes = {n: str(b.dtype).replace("torch.", "") for n, b in model.named_buffers() if n not in persistent} meta = {"quant_spec": json.dumps(quant_spec(model)), "config": json.dumps({k: v for k, v in dict(model.config).items() if not k.startswith("_")}, default=str), "buffer_dtypes": json.dumps(buf_dtypes), "format": "wanq-v1: w = (code - zero) * scale, codes packed little-endian per QuantLinear.bits"} if extra: meta.update({k: json.dumps(v) for k, v in extra.items()}) save_file(sd, str(path), metadata=meta) def load_quantized(path, device="cuda", dtype=torch.bfloat16): """Build the transformer from the saved config with empty parameters, swap in QuantLinear shells, load the packed weights.""" from accelerate import init_empty_weights from diffusers import WanTransformer3DModel from safetensors import safe_open from safetensors.torch import load_file with safe_open(str(path), framework="pt") as f: meta = f.metadata() spec, cfg = json.loads(meta["quant_spec"]), json.loads(meta["config"]) cfg = {k: v for k, v in cfg.items() if not k.startswith("_")} with init_empty_weights(): # parameters on meta, buffers (RoPE tables) computed for real model = WanTransformer3DModel.from_config(cfg) for name, (bits, group) in spec.items(): lin = model.get_submodule(name) _set_module(model, name, QuantLinear(lin.in_features, lin.out_features, bits, group, bias=lin.bias is not None, dtype=dtype, device="cpu")) sd = load_file(str(path)) missing, unexpected = model.load_state_dict(sd, strict=False, assign=True) missing = [k for k in missing if not k.endswith((".freqs_cos", ".freqs_sin", ".freqs"))] assert not missing and not unexpected, f"missing {missing[:5]} unexpected {unexpected[:5]}" for n, dt in json.loads(meta.get("buffer_dtypes", "{}")).items(): # e.g. RoPE tables: same dtype as in the model that was saved parent, _, child = n.rpartition(".") mod = model.get_submodule(parent) mod._buffers[child] = mod._buffers[child].to(getattr(torch, dt)) model = model.to(device) for m in model.modules(): # non-quantized params come back in their saved dtype (bf16) if isinstance(m, QuantLinear): m.compute_dtype = dtype return model.eval() # ----------------------------------------------------------------------------- fine-tuning helpers def add_adapters(model, rank=32, alpha=None, train_scale=True): for p in model.parameters(): p.requires_grad_(False) params = [] for m in model.modules(): if isinstance(m, QuantLinear): m.add_adapters(rank, alpha, train_scale) params += [m.lora_A, m.lora_B] + ([m.log_scale] if m.log_scale is not None else []) return params def merge_adapters(model): for m in model.modules(): if isinstance(m, QuantLinear): m.merge_adapters() return model # ----------------------------------------------------------------------------- flow matching: loss, sampler (same convention as Bora) def shift_t(u, shift): return shift * u / (1 + (shift - 1) * u) def pad_text(emb_list, length=512): """list of [Li, D] -> [B, length, D] zero-padded, the way the Wan pipeline feeds umT5 states to the transformer.""" D = emb_list[0].shape[-1] out = emb_list[0].new_zeros(len(emb_list), length, D) for i, e in enumerate(emb_list): out[i, :min(length, e.shape[0])] = e[:length] return out def wan_velocity(model, x_t, sigma, text): """Wan predicts the flow velocity (noise - x) at timestep = 1000 * sigma.""" return model(hidden_states=x_t.to(torch.bfloat16), timestep=(sigma * 1000).float(), encoder_hidden_states=text.to(torch.bfloat16), return_dict=False)[0] @torch.no_grad() def wan_sample(model, text, null_text, frames=10, height=32, width=32, steps=30, cfg=5.0, shift=3.0, generator=None): """Euler flow sampler with classifier-free guidance. text/null_text [B, 512, 4096]. Returns normalized latents [B,16,F,H,W] fp32.""" B, dev = text.shape[0], text.device x = torch.randn(B, 16, frames, height, width, device=dev, generator=generator) sig = shift_t(torch.linspace(1, 0, steps + 1, device=dev), shift) for i in range(steps): s = sig[i].expand(B) if cfg != 1.0: v = wan_velocity(model, torch.cat([x, x]), torch.cat([s, s]), torch.cat([text, null_text])).float() v_c, v_u = v.chunk(2) v = v_u + cfg * (v_c - v_u) else: v = wan_velocity(model, x, s, text).float() x = x + (sig[i + 1] - sig[i]) * v return x