Ami1ne's picture
Wan2.1-1.3B quantized to Bora size
fc74d12 verified
Raw History Blame Contribute Delete
20.3 kB
# wanq.py -- weight-only low-bit quantization for the Wan2.1 video DiT (diffusers WanTransformer3DModel), Bora part 2.
#
# Storage: every nn.Linear inside transformer.blocks becomes a QuantLinear holding
# qweight uint8, the integer codes bit-packed (2, 3, 4 or 8 bits per weight)
# scale fp16 [out, in/group] zero fp16 [out, in/group] w = (code - zero) * scale
# Weights are dequantized on the fly, one layer at a time, so GPU memory holds the packed model plus one bf16 layer.
#
# Quantizers (all asymmetric, per output row and per group of `group` input columns):
# rtn round-to-nearest with min/max range
# hqq Half-Quadratic Quantization (Badri & Shaji 2023): calibration-free, optimizes the zero point with an lp<1 proximal solver
# gptq GPTQ (Frantar et al. 2023): column-by-column rounding with error feedback through the inverse Hessian of the layer input
# Fine-tuning: QA-LoRA (Xu et al. 2024) adapters on group-pooled inputs, plus a learnable per-group log-scale. Both merge exactly
# into (scale, zero) afterwards, so the fine-tuned model has the same size as the PTQ model.
import copy
import json
import math
import time
import torch
import torch.nn as nn
import torch.nn.functional as F
# ----------------------------------------------------------------------------- bit packing
def packed_len(n, bits):
assert n % 8 == 0, f"in_features {n} must be a multiple of 8"
return n * bits // 8
def pack_codes(codes, bits):
"""codes uint8 [..., n] with values < 2**bits -> uint8 [..., n * bits / 8]."""
*lead, n = codes.shape
if bits == 8:
return codes.contiguous()
if bits == 4:
c = codes.reshape(*lead, n // 2, 2)
return (c[..., 0] | (c[..., 1] << 4)).contiguous()
if bits == 2:
c = codes.reshape(*lead, n // 4, 4)
return (c[..., 0] | (c[..., 1] << 2) | (c[..., 2] << 4) | (c[..., 3] << 6)).contiguous()
if bits == 3: # 8 codes -> 24 bits -> 3 bytes
c = codes.reshape(*lead, n // 8, 8).to(torch.int32)
shifts = torch.arange(8, device=codes.device, dtype=torch.int32) * 3
word = (c << shifts).sum(-1)
out = torch.stack([(word >> (8 * j)) & 0xFF for j in range(3)], dim=-1).to(torch.uint8)
return out.reshape(*lead, n // 8 * 3).contiguous()
raise ValueError(f"unsupported bits {bits}")
def unpack_codes(packed, bits, n):
*lead, _ = packed.shape
if bits == 8:
return packed
if bits == 4:
return torch.stack([packed & 0x0F, packed >> 4], dim=-1).reshape(*lead, n)
if bits == 2:
return torch.stack([(packed >> s) & 0x03 for s in (0, 2, 4, 6)], dim=-1).reshape(*lead, n)
if bits == 3:
b = packed.reshape(*lead, n // 8, 3).to(torch.int32)
word = b[..., 0] | (b[..., 1] << 8) | (b[..., 2] << 16)
shifts = torch.arange(8, device=packed.device, dtype=torch.int32) * 3
return ((word.unsqueeze(-1) >> shifts) & 0x07).to(torch.uint8).reshape(*lead, n)
raise ValueError(f"unsupported bits {bits}")
# ----------------------------------------------------------------------------- the quantized layer
class QuantLinear(nn.Module):
def __init__(self, in_features, out_features, bits, group, bias=True, dtype=torch.bfloat16, device=None):
super().__init__()
assert in_features % group == 0, f"in_features {in_features} not divisible by group {group}"
self.in_features, self.out_features, self.bits, self.group = in_features, out_features, bits, group
self.ngroups = in_features // group
self.register_buffer("qweight", torch.zeros(out_features, packed_len(in_features, bits), dtype=torch.uint8, device=device))
self.register_buffer("scale", torch.zeros(out_features, self.ngroups, dtype=torch.float16, device=device))
self.register_buffer("zero", torch.zeros(out_features, self.ngroups, dtype=torch.float16, device=device))
self.register_buffer("bias", torch.zeros(out_features, dtype=dtype, device=device) if bias else None)
self.compute_dtype = dtype
self.lora_A = self.lora_B = self.log_scale = None
self.lora_scaling = 0.0
self._cache = None
@classmethod
def from_codes(cls, linear, codes, scale, zero, bits, group):
q = cls(linear.in_features, linear.out_features, bits, group, bias=linear.bias is not None,
dtype=linear.weight.dtype, device=linear.weight.device)
q.qweight.copy_(pack_codes(codes.to(torch.uint8), bits))
q.scale.copy_(scale.to(torch.float16))
q.zero.copy_(zero.to(torch.float16))
if linear.bias is not None:
q.bias.copy_(linear.bias.detach())
return q
def codes(self):
return unpack_codes(self.qweight, self.bits, self.in_features)
def effective_scale(self):
if self.log_scale is None:
return self.scale
return self.scale.float() * self.log_scale.exp()
def dequant(self, dtype=None):
"""Dense weight [out, in]. Differentiable w.r.t. log_scale when that is set."""
dtype = dtype or self.compute_dtype
c = self.codes().view(self.out_features, self.ngroups, self.group)
w = (c.float() - self.zero.float().unsqueeze(-1)) * self.effective_scale().float().unsqueeze(-1) # fp32 math, one layer at a time
return w.reshape(self.out_features, self.in_features).to(dtype)
def dense_with_adapters(self):
"""fp32 weight including the adapter delta (for checking merges): W + scaling * B A / group, constant within each group."""
w = self.dequant(torch.float32)
if self.lora_A is not None:
m = (self.lora_B.float() @ self.lora_A.float()) * self.lora_scaling / self.group
w = w + m.repeat_interleave(self.group, dim=1)
return w
def cache_dense(self, on=True):
"""Keep the dense weight around (faster sweeps; uses bf16 memory). Off by default."""
self._cache = self.dequant() if on else None
def forward(self, x):
w = self._cache if self._cache is not None else self.dequant(x.dtype)
y = F.linear(x, w, self.bias)
if self.lora_A is not None:
pooled = x.reshape(*x.shape[:-1], self.ngroups, self.group).float().mean(-1) # QA-LoRA: one input per group
y = y + (F.linear(F.linear(pooled, self.lora_A), self.lora_B) * self.lora_scaling).to(y.dtype)
return y
# ---- fine-tuning hooks ----
def add_adapters(self, rank, alpha=None, train_scale=True):
dev = self.qweight.device
self.lora_A = nn.Parameter(torch.randn(rank, self.ngroups, device=dev) / math.sqrt(self.ngroups))
self.lora_B = nn.Parameter(torch.zeros(self.out_features, rank, device=dev))
self.lora_scaling = (alpha or rank) / rank
if train_scale:
self.log_scale = nn.Parameter(torch.zeros(self.out_features, self.ngroups, device=dev))
@torch.no_grad()
def merge_adapters(self):
"""Fold log_scale into scale and the QA-LoRA delta into zero. Exact up to fp16 rounding of the new scale/zero."""
s = self.effective_scale().float()
z = self.zero.float()
if self.lora_A is not None:
m = (self.lora_B.float() @ self.lora_A.float()) * self.lora_scaling # [out, ngroups]
z = z - m / (self.group * s)
self.scale.copy_(s.half())
self.zero.copy_(z.half())
self.lora_A = self.lora_B = self.log_scale = None
self.lora_scaling = 0.0
def extra_repr(self):
return f"in={self.in_features}, out={self.out_features}, bits={self.bits}, group={self.group}"
# ----------------------------------------------------------------------------- quantizers (fp32 W [out, in] -> codes, scale, zero)
def _minmax_params(wg, maxq):
mn, mx = wg.amin(-1), wg.amax(-1)
scale = ((mx - mn) / maxq).clamp(min=1e-8)
return mn, scale
def quant_rtn(w, bits, group):
maxq = 2 ** bits - 1
out, inp = w.shape
wg = w.view(out, inp // group, group)
mn, scale = _minmax_params(wg, maxq)
zero = torch.round(-mn / scale)
s16, z16 = scale.half().float(), zero.half().float()
codes = torch.clamp(torch.round(wg / s16.unsqueeze(-1) + z16.unsqueeze(-1)), 0, maxq)
return codes.reshape(out, inp).to(torch.uint8), s16, z16
def quant_hqq(w, bits, group, iters=20, lp=0.7, beta=10.0, kappa=1.01):
"""HQQ: keep the min/max scale, optimize a float zero point with a half-quadratic lp-norm (p<1) solver. No data needed."""
maxq = 2 ** bits - 1
out, inp = w.shape
wg = w.view(out, inp // group, group).float()
mn, scale = _minmax_params(wg, maxq)
inv = (1.0 / scale).unsqueeze(-1)
z = (-mn / scale).unsqueeze(-1)
def shrink(x, b):
return torch.sign(x) * F.relu(x.abs() - (1.0 / b) * x.abs().clamp(min=1e-12).pow(lp - 1))
best_err, best_z = float("inf"), z
for _ in range(iters):
wq = torch.clamp(torch.round(wg * inv + z), 0, maxq)
wr = (wq - z) / inv
err = (wg - wr).abs().mean().item()
if err < best_err:
best_err, best_z = err, z
else:
break
we = shrink(wg - wr, beta)
z = (wq - (wg - we) * inv).mean(-1, keepdim=True)
beta *= kappa
s16 = scale.half().float()
z16 = best_z.squeeze(-1).half().float()
codes = torch.clamp(torch.round(wg / s16.unsqueeze(-1) + z16.unsqueeze(-1)), 0, maxq)
return codes.reshape(out, inp).to(torch.uint8), s16, z16
def quant_gptq(w, H, bits, group, damp=0.01, blocksize=128):
"""GPTQ with groupwise min/max parameters computed on the error-updated weights. H: [in, in] = E[x x^T] of the layer input."""
maxq = 2 ** bits - 1
W = w.clone().float()
out, cols = W.shape
assert blocksize % group == 0 or group % blocksize == 0
H = H.clone().float()
dead = torch.diag(H) == 0
H[dead, dead] = 1
W[:, dead] = 0
H.diagonal().add_(damp * torch.mean(torch.diag(H)))
try:
L = torch.linalg.cholesky(H)
except RuntimeError: # rare: not positive definite -> more damping
H.diagonal().add_(0.1 * torch.mean(torch.diag(H)))
L = torch.linalg.cholesky(H)
Hinv = torch.linalg.cholesky(torch.cholesky_inverse(L), upper=True)
codes = torch.zeros(out, cols, dtype=torch.uint8, device=W.device)
scales = torch.zeros(out, cols // group, device=W.device)
zeros = torch.zeros(out, cols // group, device=W.device)
s = z = None
for i1 in range(0, cols, blocksize):
i2 = min(i1 + blocksize, cols)
W1 = W[:, i1:i2].clone()
Err1 = torch.zeros_like(W1)
Hinv1 = Hinv[i1:i2, i1:i2]
for i in range(i2 - i1):
col = i1 + i
if col % group == 0: # new group: parameters from the current (error-updated) weights
gw = W1[:, i:i + group] if i + group <= i2 - i1 else W[:, col:col + group]
mn, mx = gw.amin(1), gw.amax(1)
s = ((mx - mn) / maxq).clamp(min=1e-8).half().float()
z = torch.round(-mn / s).half().float()
scales[:, col // group], zeros[:, col // group] = s, z
wcol = W1[:, i]
q = torch.clamp(torch.round(wcol / s + z), 0, maxq)
codes[:, col] = q.to(torch.uint8)
deq = (q - z) * s
d = Hinv1[i, i]
e = (wcol - deq) / d
W1[:, i:] -= e.unsqueeze(1) @ Hinv1[i, i:].unsqueeze(0)
Err1[:, i] = e
W[:, i2:] -= Err1 @ Hinv[i1:i2, i2:]
return codes, scales, zeros
# ----------------------------------------------------------------------------- model-level helpers
def block_linears(model):
"""(name, module) for every nn.Linear inside transformer.blocks. Embedders, norms and the output head stay bf16."""
return [(n, m) for n, m in model.named_modules() if n.startswith("blocks.") and isinstance(m, nn.Linear)]
def _set_module(model, name, new):
parent, _, child = name.rpartition(".")
setattr(model.get_submodule(parent), child, new)
@torch.no_grad()
def collect_hessians(model, batches, log=print):
"""Run the bf16 model on calibration batches; accumulate H = sum x x^T / n for the input of every block linear (fp32, on GPU).
batches: iterable of dicts with hidden_states, timestep, encoder_hidden_states (already on the model device)."""
H, n, hooks = {}, {}, []
for name, mod in block_linears(model):
H[name] = torch.zeros(mod.in_features, mod.in_features, device=mod.weight.device, dtype=torch.float32)
n[name] = 0
def hook(m, inp, out, name=name):
x = inp[0].detach().reshape(-1, inp[0].shape[-1]).float()
H[name].addmm_(x.T, x)
n[name] += x.shape[0]
hooks.append(mod.register_forward_hook(hook))
t0 = time.time()
try:
for i, b in enumerate(batches):
model(hidden_states=b["hidden_states"], timestep=b["timestep"], encoder_hidden_states=b["encoder_hidden_states"], return_dict=False)
finally:
for h in hooks:
h.remove()
for k in H:
H[k] /= max(1, n[k])
gb = sum(h.numel() * 4 for h in H.values()) / 1024 ** 3
log(f"hessians: {len(H)} layers, {i + 1} batches, {time.time() - t0:.0f}s, {gb:.1f} GB")
return H
@torch.no_grad()
def quantize_model(base, method, bits, group, hessians=None, log=print, **kw):
"""Copy of `base` with every block linear replaced by a QuantLinear. `base` is not modified."""
model = copy.deepcopy(base)
t0 = time.time()
err_num = err_den = 0.0
for name, lin in block_linears(model):
w = lin.weight.detach().float()
if method == "rtn":
codes, s, z = quant_rtn(w, bits, group)
elif method == "hqq":
codes, s, z = quant_hqq(w, bits, group, **kw)
elif method == "gptq":
assert hessians is not None, "gptq needs hessians (collect_hessians)"
codes, s, z = quant_gptq(w, hessians[name], bits, group, **kw)
else:
raise ValueError(method)
q = QuantLinear.from_codes(lin, codes, s, z, bits, group)
err_num += (q.dequant(torch.float32) - w).pow(2).sum().item()
err_den += w.pow(2).sum().item()
_set_module(model, name, q)
log(f"{method} W{bits} g{group}: {len(block_linears(base))} layers in {time.time() - t0:.0f}s | relative weight error {math.sqrt(err_num / err_den):.4f}")
return model
def quant_spec(model):
return {n: [m.bits, m.group] for n, m in model.named_modules() if isinstance(m, QuantLinear)}
def model_bytes(model):
"""Bytes of everything the model stores (parameters + persistent buffers), i.e. the checkpoint size."""
return sum(t.numel() * t.element_size() for t in model.state_dict().values())
def size_report(model):
q = [m for m in model.modules() if isinstance(m, QuantLinear)]
qw = sum(m.in_features * m.out_features for m in q)
qbytes = sum(m.qweight.numel() + 2 * m.scale.numel() + 2 * m.zero.numel() for m in q)
total = model_bytes(model)
return {"total_MB": total / 1e6, "quantized_weights": qw, "quantized_MB": qbytes / 1e6, # decimal MB, like "Bora = 673 MB"
"bits_per_quantized_weight": 8 * qbytes / max(1, qw), "other_MB": (total - qbytes) / 1e6}
def save_quantized(model, path, extra=None):
from safetensors.torch import save_file
sd = {k: v.contiguous() for k, v in model.state_dict().items()}
persistent = set(model.state_dict().keys())
buf_dtypes = {n: str(b.dtype).replace("torch.", "") for n, b in model.named_buffers() if n not in persistent}
meta = {"quant_spec": json.dumps(quant_spec(model)), "config": json.dumps({k: v for k, v in dict(model.config).items() if not k.startswith("_")}, default=str), "buffer_dtypes": json.dumps(buf_dtypes),
"format": "wanq-v1: w = (code - zero) * scale, codes packed little-endian per QuantLinear.bits"}
if extra:
meta.update({k: json.dumps(v) for k, v in extra.items()})
save_file(sd, str(path), metadata=meta)
def load_quantized(path, device="cuda", dtype=torch.bfloat16):
"""Build the transformer from the saved config with empty parameters, swap in QuantLinear shells, load the packed weights."""
from accelerate import init_empty_weights
from diffusers import WanTransformer3DModel
from safetensors import safe_open
from safetensors.torch import load_file
with safe_open(str(path), framework="pt") as f:
meta = f.metadata()
spec, cfg = json.loads(meta["quant_spec"]), json.loads(meta["config"])
cfg = {k: v for k, v in cfg.items() if not k.startswith("_")}
with init_empty_weights(): # parameters on meta, buffers (RoPE tables) computed for real
model = WanTransformer3DModel.from_config(cfg)
for name, (bits, group) in spec.items():
lin = model.get_submodule(name)
_set_module(model, name, QuantLinear(lin.in_features, lin.out_features, bits, group, bias=lin.bias is not None, dtype=dtype, device="cpu"))
sd = load_file(str(path))
missing, unexpected = model.load_state_dict(sd, strict=False, assign=True)
missing = [k for k in missing if not k.endswith((".freqs_cos", ".freqs_sin", ".freqs"))]
assert not missing and not unexpected, f"missing {missing[:5]} unexpected {unexpected[:5]}"
for n, dt in json.loads(meta.get("buffer_dtypes", "{}")).items(): # e.g. RoPE tables: same dtype as in the model that was saved
parent, _, child = n.rpartition(".")
mod = model.get_submodule(parent)
mod._buffers[child] = mod._buffers[child].to(getattr(torch, dt))
model = model.to(device)
for m in model.modules(): # non-quantized params come back in their saved dtype (bf16)
if isinstance(m, QuantLinear):
m.compute_dtype = dtype
return model.eval()
# ----------------------------------------------------------------------------- fine-tuning helpers
def add_adapters(model, rank=32, alpha=None, train_scale=True):
for p in model.parameters():
p.requires_grad_(False)
params = []
for m in model.modules():
if isinstance(m, QuantLinear):
m.add_adapters(rank, alpha, train_scale)
params += [m.lora_A, m.lora_B] + ([m.log_scale] if m.log_scale is not None else [])
return params
def merge_adapters(model):
for m in model.modules():
if isinstance(m, QuantLinear):
m.merge_adapters()
return model
# ----------------------------------------------------------------------------- flow matching: loss, sampler (same convention as Bora)
def shift_t(u, shift):
return shift * u / (1 + (shift - 1) * u)
def pad_text(emb_list, length=512):
"""list of [Li, D] -> [B, length, D] zero-padded, the way the Wan pipeline feeds umT5 states to the transformer."""
D = emb_list[0].shape[-1]
out = emb_list[0].new_zeros(len(emb_list), length, D)
for i, e in enumerate(emb_list):
out[i, :min(length, e.shape[0])] = e[:length]
return out
def wan_velocity(model, x_t, sigma, text):
"""Wan predicts the flow velocity (noise - x) at timestep = 1000 * sigma."""
return model(hidden_states=x_t.to(torch.bfloat16), timestep=(sigma * 1000).float(), encoder_hidden_states=text.to(torch.bfloat16),
return_dict=False)[0]
@torch.no_grad()
def wan_sample(model, text, null_text, frames=10, height=32, width=32, steps=30, cfg=5.0, shift=3.0, generator=None):
"""Euler flow sampler with classifier-free guidance. text/null_text [B, 512, 4096]. Returns normalized latents [B,16,F,H,W] fp32."""
B, dev = text.shape[0], text.device
x = torch.randn(B, 16, frames, height, width, device=dev, generator=generator)
sig = shift_t(torch.linspace(1, 0, steps + 1, device=dev), shift)
for i in range(steps):
s = sig[i].expand(B)
if cfg != 1.0:
v = wan_velocity(model, torch.cat([x, x]), torch.cat([s, s]), torch.cat([text, null_text])).float()
v_c, v_u = v.chunk(2)
v = v_u + cfg * (v_c - v_u)
else:
v = wan_velocity(model, x, s, text).float()
x = x + (sig[i + 1] - sig[i]) * v
return x