Download code/wanq.py from Ami1ne/wan1.3b-bora-quant: direct link, hf CLI and curl.
- Browser
- Download file 20.3 kB
-
https://huggingface.co/Ami1ne/wan1.3b-bora-quant/resolve/main/code/wanq.py
- Command line
-
hf download hf://Ami1ne/wan1.3b-bora-quant/code/wanq.py
-
curl -L -o wanq.py https://huggingface.co/Ami1ne/wan1.3b-bora-quant/resolve/main/code/wanq.py
20.3 kB
| # wanq.py -- weight-only low-bit quantization for the Wan2.1 video DiT (diffusers WanTransformer3DModel), Bora part 2. | |
| # | |
| # Storage: every nn.Linear inside transformer.blocks becomes a QuantLinear holding | |
| # qweight uint8, the integer codes bit-packed (2, 3, 4 or 8 bits per weight) | |
| # scale fp16 [out, in/group] zero fp16 [out, in/group] w = (code - zero) * scale | |
| # Weights are dequantized on the fly, one layer at a time, so GPU memory holds the packed model plus one bf16 layer. | |
| # | |
| # Quantizers (all asymmetric, per output row and per group of `group` input columns): | |
| # rtn round-to-nearest with min/max range | |
| # hqq Half-Quadratic Quantization (Badri & Shaji 2023): calibration-free, optimizes the zero point with an lp<1 proximal solver | |
| # gptq GPTQ (Frantar et al. 2023): column-by-column rounding with error feedback through the inverse Hessian of the layer input | |
| # Fine-tuning: QA-LoRA (Xu et al. 2024) adapters on group-pooled inputs, plus a learnable per-group log-scale. Both merge exactly | |
| # into (scale, zero) afterwards, so the fine-tuned model has the same size as the PTQ model. | |
| import copy | |
| import json | |
| import math | |
| import time | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| # ----------------------------------------------------------------------------- bit packing | |
| def packed_len(n, bits): | |
| assert n % 8 == 0, f"in_features {n} must be a multiple of 8" | |
| return n * bits // 8 | |
| def pack_codes(codes, bits): | |
| """codes uint8 [..., n] with values < 2**bits -> uint8 [..., n * bits / 8].""" | |
| *lead, n = codes.shape | |
| if bits == 8: | |
| return codes.contiguous() | |
| if bits == 4: | |
| c = codes.reshape(*lead, n // 2, 2) | |
| return (c[..., 0] | (c[..., 1] << 4)).contiguous() | |
| if bits == 2: | |
| c = codes.reshape(*lead, n // 4, 4) | |
| return (c[..., 0] | (c[..., 1] << 2) | (c[..., 2] << 4) | (c[..., 3] << 6)).contiguous() | |
| if bits == 3: # 8 codes -> 24 bits -> 3 bytes | |
| c = codes.reshape(*lead, n // 8, 8).to(torch.int32) | |
| shifts = torch.arange(8, device=codes.device, dtype=torch.int32) * 3 | |
| word = (c << shifts).sum(-1) | |
| out = torch.stack([(word >> (8 * j)) & 0xFF for j in range(3)], dim=-1).to(torch.uint8) | |
| return out.reshape(*lead, n // 8 * 3).contiguous() | |
| raise ValueError(f"unsupported bits {bits}") | |
| def unpack_codes(packed, bits, n): | |
| *lead, _ = packed.shape | |
| if bits == 8: | |
| return packed | |
| if bits == 4: | |
| return torch.stack([packed & 0x0F, packed >> 4], dim=-1).reshape(*lead, n) | |
| if bits == 2: | |
| return torch.stack([(packed >> s) & 0x03 for s in (0, 2, 4, 6)], dim=-1).reshape(*lead, n) | |
| if bits == 3: | |
| b = packed.reshape(*lead, n // 8, 3).to(torch.int32) | |
| word = b[..., 0] | (b[..., 1] << 8) | (b[..., 2] << 16) | |
| shifts = torch.arange(8, device=packed.device, dtype=torch.int32) * 3 | |
| return ((word.unsqueeze(-1) >> shifts) & 0x07).to(torch.uint8).reshape(*lead, n) | |
| raise ValueError(f"unsupported bits {bits}") | |
| # ----------------------------------------------------------------------------- the quantized layer | |
| class QuantLinear(nn.Module): | |
| def __init__(self, in_features, out_features, bits, group, bias=True, dtype=torch.bfloat16, device=None): | |
| super().__init__() | |
| assert in_features % group == 0, f"in_features {in_features} not divisible by group {group}" | |
| self.in_features, self.out_features, self.bits, self.group = in_features, out_features, bits, group | |
| self.ngroups = in_features // group | |
| self.register_buffer("qweight", torch.zeros(out_features, packed_len(in_features, bits), dtype=torch.uint8, device=device)) | |
| self.register_buffer("scale", torch.zeros(out_features, self.ngroups, dtype=torch.float16, device=device)) | |
| self.register_buffer("zero", torch.zeros(out_features, self.ngroups, dtype=torch.float16, device=device)) | |
| self.register_buffer("bias", torch.zeros(out_features, dtype=dtype, device=device) if bias else None) | |
| self.compute_dtype = dtype | |
| self.lora_A = self.lora_B = self.log_scale = None | |
| self.lora_scaling = 0.0 | |
| self._cache = None | |
| def from_codes(cls, linear, codes, scale, zero, bits, group): | |
| q = cls(linear.in_features, linear.out_features, bits, group, bias=linear.bias is not None, | |
| dtype=linear.weight.dtype, device=linear.weight.device) | |
| q.qweight.copy_(pack_codes(codes.to(torch.uint8), bits)) | |
| q.scale.copy_(scale.to(torch.float16)) | |
| q.zero.copy_(zero.to(torch.float16)) | |
| if linear.bias is not None: | |
| q.bias.copy_(linear.bias.detach()) | |
| return q | |
| def codes(self): | |
| return unpack_codes(self.qweight, self.bits, self.in_features) | |
| def effective_scale(self): | |
| if self.log_scale is None: | |
| return self.scale | |
| return self.scale.float() * self.log_scale.exp() | |
| def dequant(self, dtype=None): | |
| """Dense weight [out, in]. Differentiable w.r.t. log_scale when that is set.""" | |
| dtype = dtype or self.compute_dtype | |
| c = self.codes().view(self.out_features, self.ngroups, self.group) | |
| w = (c.float() - self.zero.float().unsqueeze(-1)) * self.effective_scale().float().unsqueeze(-1) # fp32 math, one layer at a time | |
| return w.reshape(self.out_features, self.in_features).to(dtype) | |
| def dense_with_adapters(self): | |
| """fp32 weight including the adapter delta (for checking merges): W + scaling * B A / group, constant within each group.""" | |
| w = self.dequant(torch.float32) | |
| if self.lora_A is not None: | |
| m = (self.lora_B.float() @ self.lora_A.float()) * self.lora_scaling / self.group | |
| w = w + m.repeat_interleave(self.group, dim=1) | |
| return w | |
| def cache_dense(self, on=True): | |
| """Keep the dense weight around (faster sweeps; uses bf16 memory). Off by default.""" | |
| self._cache = self.dequant() if on else None | |
| def forward(self, x): | |
| w = self._cache if self._cache is not None else self.dequant(x.dtype) | |
| y = F.linear(x, w, self.bias) | |
| if self.lora_A is not None: | |
| pooled = x.reshape(*x.shape[:-1], self.ngroups, self.group).float().mean(-1) # QA-LoRA: one input per group | |
| y = y + (F.linear(F.linear(pooled, self.lora_A), self.lora_B) * self.lora_scaling).to(y.dtype) | |
| return y | |
| # ---- fine-tuning hooks ---- | |
| def add_adapters(self, rank, alpha=None, train_scale=True): | |
| dev = self.qweight.device | |
| self.lora_A = nn.Parameter(torch.randn(rank, self.ngroups, device=dev) / math.sqrt(self.ngroups)) | |
| self.lora_B = nn.Parameter(torch.zeros(self.out_features, rank, device=dev)) | |
| self.lora_scaling = (alpha or rank) / rank | |
| if train_scale: | |
| self.log_scale = nn.Parameter(torch.zeros(self.out_features, self.ngroups, device=dev)) | |
| def merge_adapters(self): | |
| """Fold log_scale into scale and the QA-LoRA delta into zero. Exact up to fp16 rounding of the new scale/zero.""" | |
| s = self.effective_scale().float() | |
| z = self.zero.float() | |
| if self.lora_A is not None: | |
| m = (self.lora_B.float() @ self.lora_A.float()) * self.lora_scaling # [out, ngroups] | |
| z = z - m / (self.group * s) | |
| self.scale.copy_(s.half()) | |
| self.zero.copy_(z.half()) | |
| self.lora_A = self.lora_B = self.log_scale = None | |
| self.lora_scaling = 0.0 | |
| def extra_repr(self): | |
| return f"in={self.in_features}, out={self.out_features}, bits={self.bits}, group={self.group}" | |
| # ----------------------------------------------------------------------------- quantizers (fp32 W [out, in] -> codes, scale, zero) | |
| def _minmax_params(wg, maxq): | |
| mn, mx = wg.amin(-1), wg.amax(-1) | |
| scale = ((mx - mn) / maxq).clamp(min=1e-8) | |
| return mn, scale | |
| def quant_rtn(w, bits, group): | |
| maxq = 2 ** bits - 1 | |
| out, inp = w.shape | |
| wg = w.view(out, inp // group, group) | |
| mn, scale = _minmax_params(wg, maxq) | |
| zero = torch.round(-mn / scale) | |
| s16, z16 = scale.half().float(), zero.half().float() | |
| codes = torch.clamp(torch.round(wg / s16.unsqueeze(-1) + z16.unsqueeze(-1)), 0, maxq) | |
| return codes.reshape(out, inp).to(torch.uint8), s16, z16 | |
| def quant_hqq(w, bits, group, iters=20, lp=0.7, beta=10.0, kappa=1.01): | |
| """HQQ: keep the min/max scale, optimize a float zero point with a half-quadratic lp-norm (p<1) solver. No data needed.""" | |
| maxq = 2 ** bits - 1 | |
| out, inp = w.shape | |
| wg = w.view(out, inp // group, group).float() | |
| mn, scale = _minmax_params(wg, maxq) | |
| inv = (1.0 / scale).unsqueeze(-1) | |
| z = (-mn / scale).unsqueeze(-1) | |
| def shrink(x, b): | |
| return torch.sign(x) * F.relu(x.abs() - (1.0 / b) * x.abs().clamp(min=1e-12).pow(lp - 1)) | |
| best_err, best_z = float("inf"), z | |
| for _ in range(iters): | |
| wq = torch.clamp(torch.round(wg * inv + z), 0, maxq) | |
| wr = (wq - z) / inv | |
| err = (wg - wr).abs().mean().item() | |
| if err < best_err: | |
| best_err, best_z = err, z | |
| else: | |
| break | |
| we = shrink(wg - wr, beta) | |
| z = (wq - (wg - we) * inv).mean(-1, keepdim=True) | |
| beta *= kappa | |
| s16 = scale.half().float() | |
| z16 = best_z.squeeze(-1).half().float() | |
| codes = torch.clamp(torch.round(wg / s16.unsqueeze(-1) + z16.unsqueeze(-1)), 0, maxq) | |
| return codes.reshape(out, inp).to(torch.uint8), s16, z16 | |
| def quant_gptq(w, H, bits, group, damp=0.01, blocksize=128): | |
| """GPTQ with groupwise min/max parameters computed on the error-updated weights. H: [in, in] = E[x x^T] of the layer input.""" | |
| maxq = 2 ** bits - 1 | |
| W = w.clone().float() | |
| out, cols = W.shape | |
| assert blocksize % group == 0 or group % blocksize == 0 | |
| H = H.clone().float() | |
| dead = torch.diag(H) == 0 | |
| H[dead, dead] = 1 | |
| W[:, dead] = 0 | |
| H.diagonal().add_(damp * torch.mean(torch.diag(H))) | |
| try: | |
| L = torch.linalg.cholesky(H) | |
| except RuntimeError: # rare: not positive definite -> more damping | |
| H.diagonal().add_(0.1 * torch.mean(torch.diag(H))) | |
| L = torch.linalg.cholesky(H) | |
| Hinv = torch.linalg.cholesky(torch.cholesky_inverse(L), upper=True) | |
| codes = torch.zeros(out, cols, dtype=torch.uint8, device=W.device) | |
| scales = torch.zeros(out, cols // group, device=W.device) | |
| zeros = torch.zeros(out, cols // group, device=W.device) | |
| s = z = None | |
| for i1 in range(0, cols, blocksize): | |
| i2 = min(i1 + blocksize, cols) | |
| W1 = W[:, i1:i2].clone() | |
| Err1 = torch.zeros_like(W1) | |
| Hinv1 = Hinv[i1:i2, i1:i2] | |
| for i in range(i2 - i1): | |
| col = i1 + i | |
| if col % group == 0: # new group: parameters from the current (error-updated) weights | |
| gw = W1[:, i:i + group] if i + group <= i2 - i1 else W[:, col:col + group] | |
| mn, mx = gw.amin(1), gw.amax(1) | |
| s = ((mx - mn) / maxq).clamp(min=1e-8).half().float() | |
| z = torch.round(-mn / s).half().float() | |
| scales[:, col // group], zeros[:, col // group] = s, z | |
| wcol = W1[:, i] | |
| q = torch.clamp(torch.round(wcol / s + z), 0, maxq) | |
| codes[:, col] = q.to(torch.uint8) | |
| deq = (q - z) * s | |
| d = Hinv1[i, i] | |
| e = (wcol - deq) / d | |
| W1[:, i:] -= e.unsqueeze(1) @ Hinv1[i, i:].unsqueeze(0) | |
| Err1[:, i] = e | |
| W[:, i2:] -= Err1 @ Hinv[i1:i2, i2:] | |
| return codes, scales, zeros | |
| # ----------------------------------------------------------------------------- model-level helpers | |
| def block_linears(model): | |
| """(name, module) for every nn.Linear inside transformer.blocks. Embedders, norms and the output head stay bf16.""" | |
| return [(n, m) for n, m in model.named_modules() if n.startswith("blocks.") and isinstance(m, nn.Linear)] | |
| def _set_module(model, name, new): | |
| parent, _, child = name.rpartition(".") | |
| setattr(model.get_submodule(parent), child, new) | |
| def collect_hessians(model, batches, log=print): | |
| """Run the bf16 model on calibration batches; accumulate H = sum x x^T / n for the input of every block linear (fp32, on GPU). | |
| batches: iterable of dicts with hidden_states, timestep, encoder_hidden_states (already on the model device).""" | |
| H, n, hooks = {}, {}, [] | |
| for name, mod in block_linears(model): | |
| H[name] = torch.zeros(mod.in_features, mod.in_features, device=mod.weight.device, dtype=torch.float32) | |
| n[name] = 0 | |
| def hook(m, inp, out, name=name): | |
| x = inp[0].detach().reshape(-1, inp[0].shape[-1]).float() | |
| H[name].addmm_(x.T, x) | |
| n[name] += x.shape[0] | |
| hooks.append(mod.register_forward_hook(hook)) | |
| t0 = time.time() | |
| try: | |
| for i, b in enumerate(batches): | |
| model(hidden_states=b["hidden_states"], timestep=b["timestep"], encoder_hidden_states=b["encoder_hidden_states"], return_dict=False) | |
| finally: | |
| for h in hooks: | |
| h.remove() | |
| for k in H: | |
| H[k] /= max(1, n[k]) | |
| gb = sum(h.numel() * 4 for h in H.values()) / 1024 ** 3 | |
| log(f"hessians: {len(H)} layers, {i + 1} batches, {time.time() - t0:.0f}s, {gb:.1f} GB") | |
| return H | |
| def quantize_model(base, method, bits, group, hessians=None, log=print, **kw): | |
| """Copy of `base` with every block linear replaced by a QuantLinear. `base` is not modified.""" | |
| model = copy.deepcopy(base) | |
| t0 = time.time() | |
| err_num = err_den = 0.0 | |
| for name, lin in block_linears(model): | |
| w = lin.weight.detach().float() | |
| if method == "rtn": | |
| codes, s, z = quant_rtn(w, bits, group) | |
| elif method == "hqq": | |
| codes, s, z = quant_hqq(w, bits, group, **kw) | |
| elif method == "gptq": | |
| assert hessians is not None, "gptq needs hessians (collect_hessians)" | |
| codes, s, z = quant_gptq(w, hessians[name], bits, group, **kw) | |
| else: | |
| raise ValueError(method) | |
| q = QuantLinear.from_codes(lin, codes, s, z, bits, group) | |
| err_num += (q.dequant(torch.float32) - w).pow(2).sum().item() | |
| err_den += w.pow(2).sum().item() | |
| _set_module(model, name, q) | |
| log(f"{method} W{bits} g{group}: {len(block_linears(base))} layers in {time.time() - t0:.0f}s | relative weight error {math.sqrt(err_num / err_den):.4f}") | |
| return model | |
| def quant_spec(model): | |
| return {n: [m.bits, m.group] for n, m in model.named_modules() if isinstance(m, QuantLinear)} | |
| def model_bytes(model): | |
| """Bytes of everything the model stores (parameters + persistent buffers), i.e. the checkpoint size.""" | |
| return sum(t.numel() * t.element_size() for t in model.state_dict().values()) | |
| def size_report(model): | |
| q = [m for m in model.modules() if isinstance(m, QuantLinear)] | |
| qw = sum(m.in_features * m.out_features for m in q) | |
| qbytes = sum(m.qweight.numel() + 2 * m.scale.numel() + 2 * m.zero.numel() for m in q) | |
| total = model_bytes(model) | |
| return {"total_MB": total / 1e6, "quantized_weights": qw, "quantized_MB": qbytes / 1e6, # decimal MB, like "Bora = 673 MB" | |
| "bits_per_quantized_weight": 8 * qbytes / max(1, qw), "other_MB": (total - qbytes) / 1e6} | |
| def save_quantized(model, path, extra=None): | |
| from safetensors.torch import save_file | |
| sd = {k: v.contiguous() for k, v in model.state_dict().items()} | |
| persistent = set(model.state_dict().keys()) | |
| buf_dtypes = {n: str(b.dtype).replace("torch.", "") for n, b in model.named_buffers() if n not in persistent} | |
| meta = {"quant_spec": json.dumps(quant_spec(model)), "config": json.dumps({k: v for k, v in dict(model.config).items() if not k.startswith("_")}, default=str), "buffer_dtypes": json.dumps(buf_dtypes), | |
| "format": "wanq-v1: w = (code - zero) * scale, codes packed little-endian per QuantLinear.bits"} | |
| if extra: | |
| meta.update({k: json.dumps(v) for k, v in extra.items()}) | |
| save_file(sd, str(path), metadata=meta) | |
| def load_quantized(path, device="cuda", dtype=torch.bfloat16): | |
| """Build the transformer from the saved config with empty parameters, swap in QuantLinear shells, load the packed weights.""" | |
| from accelerate import init_empty_weights | |
| from diffusers import WanTransformer3DModel | |
| from safetensors import safe_open | |
| from safetensors.torch import load_file | |
| with safe_open(str(path), framework="pt") as f: | |
| meta = f.metadata() | |
| spec, cfg = json.loads(meta["quant_spec"]), json.loads(meta["config"]) | |
| cfg = {k: v for k, v in cfg.items() if not k.startswith("_")} | |
| with init_empty_weights(): # parameters on meta, buffers (RoPE tables) computed for real | |
| model = WanTransformer3DModel.from_config(cfg) | |
| for name, (bits, group) in spec.items(): | |
| lin = model.get_submodule(name) | |
| _set_module(model, name, QuantLinear(lin.in_features, lin.out_features, bits, group, bias=lin.bias is not None, dtype=dtype, device="cpu")) | |
| sd = load_file(str(path)) | |
| missing, unexpected = model.load_state_dict(sd, strict=False, assign=True) | |
| missing = [k for k in missing if not k.endswith((".freqs_cos", ".freqs_sin", ".freqs"))] | |
| assert not missing and not unexpected, f"missing {missing[:5]} unexpected {unexpected[:5]}" | |
| for n, dt in json.loads(meta.get("buffer_dtypes", "{}")).items(): # e.g. RoPE tables: same dtype as in the model that was saved | |
| parent, _, child = n.rpartition(".") | |
| mod = model.get_submodule(parent) | |
| mod._buffers[child] = mod._buffers[child].to(getattr(torch, dt)) | |
| model = model.to(device) | |
| for m in model.modules(): # non-quantized params come back in their saved dtype (bf16) | |
| if isinstance(m, QuantLinear): | |
| m.compute_dtype = dtype | |
| return model.eval() | |
| # ----------------------------------------------------------------------------- fine-tuning helpers | |
| def add_adapters(model, rank=32, alpha=None, train_scale=True): | |
| for p in model.parameters(): | |
| p.requires_grad_(False) | |
| params = [] | |
| for m in model.modules(): | |
| if isinstance(m, QuantLinear): | |
| m.add_adapters(rank, alpha, train_scale) | |
| params += [m.lora_A, m.lora_B] + ([m.log_scale] if m.log_scale is not None else []) | |
| return params | |
| def merge_adapters(model): | |
| for m in model.modules(): | |
| if isinstance(m, QuantLinear): | |
| m.merge_adapters() | |
| return model | |
| # ----------------------------------------------------------------------------- flow matching: loss, sampler (same convention as Bora) | |
| def shift_t(u, shift): | |
| return shift * u / (1 + (shift - 1) * u) | |
| def pad_text(emb_list, length=512): | |
| """list of [Li, D] -> [B, length, D] zero-padded, the way the Wan pipeline feeds umT5 states to the transformer.""" | |
| D = emb_list[0].shape[-1] | |
| out = emb_list[0].new_zeros(len(emb_list), length, D) | |
| for i, e in enumerate(emb_list): | |
| out[i, :min(length, e.shape[0])] = e[:length] | |
| return out | |
| def wan_velocity(model, x_t, sigma, text): | |
| """Wan predicts the flow velocity (noise - x) at timestep = 1000 * sigma.""" | |
| return model(hidden_states=x_t.to(torch.bfloat16), timestep=(sigma * 1000).float(), encoder_hidden_states=text.to(torch.bfloat16), | |
| return_dict=False)[0] | |
| def wan_sample(model, text, null_text, frames=10, height=32, width=32, steps=30, cfg=5.0, shift=3.0, generator=None): | |
| """Euler flow sampler with classifier-free guidance. text/null_text [B, 512, 4096]. Returns normalized latents [B,16,F,H,W] fp32.""" | |
| B, dev = text.shape[0], text.device | |
| x = torch.randn(B, 16, frames, height, width, device=dev, generator=generator) | |
| sig = shift_t(torch.linspace(1, 0, steps + 1, device=dev), shift) | |
| for i in range(steps): | |
| s = sig[i].expand(B) | |
| if cfg != 1.0: | |
| v = wan_velocity(model, torch.cat([x, x]), torch.cat([s, s]), torch.cat([text, null_text])).float() | |
| v_c, v_u = v.chunk(2) | |
| v = v_u + cfg * (v_c - v_u) | |
| else: | |
| v = wan_velocity(model, x, s, text).float() | |
| x = x + (sig[i + 1] - sig[i]) * v | |
| return x | |