"""Fast ConvRot INT8 / W6A8 / W4A8 runtime loader for zai-org/GLM-Image using comfy-kitchen.""" from __future__ import annotations import json import os import time from typing import Optional import torch import torch.nn as nn from safetensors import safe_open from safetensors.torch import load_file import comfy_kitchen as ck from diffusers import GlmImagePipeline def _decode_comfy_quant(tensor: torch.Tensor) -> dict: return json.loads(bytes(tensor.cpu().numpy().tolist()).decode("utf-8")) class ConvRotInt8Linear(nn.Module): """Direct comfy-kitchen ConvRot INT8 linear layer (bypasses Tensor subclass dispatch overhead).""" def __init__( self, weight: torch.Tensor, weight_scale: torch.Tensor, bias: Optional[torch.Tensor] = None, convrot: bool = True, convrot_groupsize: int = 256, ): super().__init__() self.register_buffer("weight", weight.contiguous()) self.register_buffer("weight_scale", weight_scale.to(torch.float32).contiguous()) if bias is not None: self.register_buffer("bias", bias.contiguous()) else: self.bias = None self.convrot = bool(convrot) self.convrot_groupsize = int(convrot_groupsize) self.out_features, self.in_features = int(weight.shape[0]), int(weight.shape[1]) def forward(self, x: torch.Tensor) -> torch.Tensor: orig_shape = x.shape x2d = x.reshape(-1, orig_shape[-1]) bias = self.bias.to(dtype=x.dtype) if (self.bias is not None and self.bias.dtype != x.dtype) else self.bias out = ck.int8_linear( x2d, self.weight, self.weight_scale, bias=bias, out_dtype=x.dtype, convrot=self.convrot, convrot_groupsize=self.convrot_groupsize, ) return out.reshape(*orig_shape[:-1], self.out_features) class ConvRotPackedLinear(nn.Module): """Direct comfy-kitchen ConvRot W6A8 (w6a8_int8) / W4A8 (asym_w4a8_int8) linear layer.""" def __init__( self, qdata: torch.Tensor, s_rel: torch.Tensor, s_channel: torch.Tensor, codebook: Optional[torch.Tensor] = None, bias: Optional[torch.Tensor] = None, group_size: int = 16, convrot_groupsize: int = 256, ): super().__init__() self.register_buffer("qdata", qdata.contiguous()) self.register_buffer("s_rel", s_rel.contiguous()) self.register_buffer("s_channel", s_channel.to(torch.float32).contiguous()) if codebook is not None: self.register_buffer("codebook", codebook.to(torch.float32).contiguous()) else: self.codebook = None if bias is not None: self.register_buffer("bias", bias.contiguous()) else: self.bias = None self.group_size = int(group_size) self.convrot_groupsize = int(convrot_groupsize) self.out_features = int(s_channel.shape[0]) self.in_features = int(s_rel.shape[1]) * self.group_size def forward(self, x: torch.Tensor) -> torch.Tensor: orig_shape = x.shape x2d = x.reshape(-1, orig_shape[-1]) bias = self.bias.to(dtype=x.dtype) if (self.bias is not None and self.bias.dtype != x.dtype) else self.bias out = ck.w4a8_int8_linear( x2d, self.qdata, self.s_rel, self.s_channel, codebook=self.codebook, bias=bias, group_size=self.group_size, convrot_groupsize=self.convrot_groupsize, out_dtype=x.dtype, ) return out.reshape(*orig_shape[:-1], self.out_features) def _set_submodule(root: nn.Module, name: str, new_mod: nn.Module) -> None: parts = name.split(".") parent = root for p in parts[:-1]: parent = getattr(parent, p) setattr(parent, parts[-1], new_mod) def apply_convrot_state_dict( module: nn.Module, safetensors_path: str, device: str = "cuda", dtype: torch.dtype = torch.bfloat16, ) -> dict: """Load a ConvRot safetensors file into a PyTorch module, replacing quantized nn.Linear layers.""" sd = load_file(safetensors_path, device="cpu") quant_prefixes = sorted(k[:-len(".comfy_quant")] for k in sd.keys() if k.endswith(".comfy_quant")) quant_set = set(quant_prefixes) # 1. Load all non-quantized parameters/buffers first regular_sd = {} for k, v in sd.items(): if k.endswith((".comfy_quant", ".weight_scale", ".weight_s_rel", ".weight_s_channel", ".weight_codebook")): continue if k.endswith(".weight") and k[:-7] in quant_set: continue if k.endswith(".bias") and k[:-5] in quant_set: continue regular_sd[k] = v missing, unexpected = module.load_state_dict(regular_sd, strict=False, assign=True) # 2. Replace each quantized Linear with ConvRotInt8Linear or ConvRotPackedLinear counts = {"int8": 0, "w6a8_int8": 0, "asym_w4a8_int8": 0} for prefix in quant_prefixes: conf = _decode_comfy_quant(sd[f"{prefix}.comfy_quant"]) fmt = conf["format"] bias = sd.get(f"{prefix}.bias", None) if bias is not None: bias = bias.to(device=device, dtype=dtype) if fmt == "int8_tensorwise": layer = ConvRotInt8Linear( weight=sd[f"{prefix}.weight"].to(device=device), weight_scale=sd[f"{prefix}.weight_scale"].to(device=device), bias=bias, convrot=conf.get("convrot", True), convrot_groupsize=conf.get("convrot_groupsize", 256), ) counts["int8"] += 1 elif fmt in ("w6a8_int8", "asym_w4a8_int8"): cb = sd.get(f"{prefix}.weight_codebook", None) layer = ConvRotPackedLinear( qdata=sd[f"{prefix}.weight"].to(device=device), s_rel=sd[f"{prefix}.weight_s_rel"].to(device=device), s_channel=sd[f"{prefix}.weight_s_channel"].to(device=device), codebook=cb.to(device=device) if cb is not None else None, bias=bias, group_size=conf.get("group_size", 16), convrot_groupsize=conf.get("convrot_groupsize", 256), ) counts[fmt] += 1 else: raise ValueError(f"Unsupported comfy_quant format: {fmt}") _set_submodule(module, prefix, layer) # Materialize any non-persistent rotary_emb buffers still on meta for m_name, mod in list(module.named_modules()): if any(b.device.type == "meta" for b in mod.buffers(recurse=False)): if hasattr(mod, "config"): _set_submodule(module, m_name, mod.__class__(config=mod.config).to(device)) module.to(device=device) del sd, regular_sd return {"counts": counts, "missing_nonquant": [m for m in missing if m[:-7] not in quant_set and m[:-5] not in quant_set]} def load_glm_image_convrot( base_model: str, transformer_path: Optional[str] = None, vlm_path: Optional[str] = None, device: str = "cuda", dtype: torch.dtype = torch.bfloat16, ) -> tuple[GlmImagePipeline, dict]: """Load GlmImagePipeline with ConvRot quantized transformer and/or vision_language_encoder.""" t0 = time.perf_counter() from diffusers import GlmImageTransformer2DModel from transformers import GlmImageForConditionalGeneration, AutoConfig kwargs = {"dtype": dtype} info = {} if transformer_path is not None: tcfg_path = os.path.join(base_model, "transformer", "config.json") tcfg = json.load(open(tcfg_path, "r", encoding="utf-8")) with torch.device("meta"): transformer = GlmImageTransformer2DModel.from_config(tcfg) info["transformer"] = apply_convrot_state_dict(transformer, transformer_path, device=device, dtype=dtype) kwargs["transformer"] = transformer if vlm_path is not None: vcfg = AutoConfig.from_pretrained(os.path.join(base_model, "vision_language_encoder")) with torch.device("meta"): vlm = GlmImageForConditionalGeneration(vcfg) info["vision_language_encoder"] = apply_convrot_state_dict(vlm, vlm_path, device=device, dtype=dtype) kwargs["vision_language_encoder"] = vlm pipe = GlmImagePipeline.from_pretrained(base_model, **kwargs) pipe.to(device) torch.cuda.synchronize() info["load_s"] = round(time.perf_counter() - t0, 3) return pipe, info