Instructions to use SUSSYEATING/GLM-Image-ConvRot with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Diffusers
How to use SUSSYEATING/GLM-Image-ConvRot with Diffusers:
pip install -U diffusers transformers accelerate
import torch from diffusers import DiffusionPipeline # switch to "mps" for apple devices pipe = DiffusionPipeline.from_pretrained("SUSSYEATING/GLM-Image-ConvRot", dtype=torch.bfloat16, device_map="cuda") prompt = "Astronaut in a jungle, cold color palette, muted colors, detailed, 8k" image = pipe(prompt).images[0] - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- Draw Things
- DiffusionBee
Download glm_image_convrot.py from SUSSYEATING/GLM-Image-ConvRot: direct link, hf CLI and curl.
- Browser
- Download file 8.55 kB
-
https://huggingface.co/SUSSYEATING/GLM-Image-ConvRot/resolve/main/glm_image_convrot.py
- Command line
-
hf download hf://SUSSYEATING/GLM-Image-ConvRot/glm_image_convrot.py
-
curl -L -o glm_image_convrot.py https://huggingface.co/SUSSYEATING/GLM-Image-ConvRot/resolve/main/glm_image_convrot.py
8.55 kB
| """Fast ConvRot INT8 / W6A8 / W4A8 runtime loader for zai-org/GLM-Image using comfy-kitchen.""" | |
| from __future__ import annotations | |
| import json | |
| import os | |
| import time | |
| from typing import Optional | |
| import torch | |
| import torch.nn as nn | |
| from safetensors import safe_open | |
| from safetensors.torch import load_file | |
| import comfy_kitchen as ck | |
| from diffusers import GlmImagePipeline | |
| def _decode_comfy_quant(tensor: torch.Tensor) -> dict: | |
| return json.loads(bytes(tensor.cpu().numpy().tolist()).decode("utf-8")) | |
| class ConvRotInt8Linear(nn.Module): | |
| """Direct comfy-kitchen ConvRot INT8 linear layer (bypasses Tensor subclass dispatch overhead).""" | |
| def __init__( | |
| self, | |
| weight: torch.Tensor, | |
| weight_scale: torch.Tensor, | |
| bias: Optional[torch.Tensor] = None, | |
| convrot: bool = True, | |
| convrot_groupsize: int = 256, | |
| ): | |
| super().__init__() | |
| self.register_buffer("weight", weight.contiguous()) | |
| self.register_buffer("weight_scale", weight_scale.to(torch.float32).contiguous()) | |
| if bias is not None: | |
| self.register_buffer("bias", bias.contiguous()) | |
| else: | |
| self.bias = None | |
| self.convrot = bool(convrot) | |
| self.convrot_groupsize = int(convrot_groupsize) | |
| self.out_features, self.in_features = int(weight.shape[0]), int(weight.shape[1]) | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: | |
| orig_shape = x.shape | |
| x2d = x.reshape(-1, orig_shape[-1]) | |
| bias = self.bias.to(dtype=x.dtype) if (self.bias is not None and self.bias.dtype != x.dtype) else self.bias | |
| out = ck.int8_linear( | |
| x2d, | |
| self.weight, | |
| self.weight_scale, | |
| bias=bias, | |
| out_dtype=x.dtype, | |
| convrot=self.convrot, | |
| convrot_groupsize=self.convrot_groupsize, | |
| ) | |
| return out.reshape(*orig_shape[:-1], self.out_features) | |
| class ConvRotPackedLinear(nn.Module): | |
| """Direct comfy-kitchen ConvRot W6A8 (w6a8_int8) / W4A8 (asym_w4a8_int8) linear layer.""" | |
| def __init__( | |
| self, | |
| qdata: torch.Tensor, | |
| s_rel: torch.Tensor, | |
| s_channel: torch.Tensor, | |
| codebook: Optional[torch.Tensor] = None, | |
| bias: Optional[torch.Tensor] = None, | |
| group_size: int = 16, | |
| convrot_groupsize: int = 256, | |
| ): | |
| super().__init__() | |
| self.register_buffer("qdata", qdata.contiguous()) | |
| self.register_buffer("s_rel", s_rel.contiguous()) | |
| self.register_buffer("s_channel", s_channel.to(torch.float32).contiguous()) | |
| if codebook is not None: | |
| self.register_buffer("codebook", codebook.to(torch.float32).contiguous()) | |
| else: | |
| self.codebook = None | |
| if bias is not None: | |
| self.register_buffer("bias", bias.contiguous()) | |
| else: | |
| self.bias = None | |
| self.group_size = int(group_size) | |
| self.convrot_groupsize = int(convrot_groupsize) | |
| self.out_features = int(s_channel.shape[0]) | |
| self.in_features = int(s_rel.shape[1]) * self.group_size | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: | |
| orig_shape = x.shape | |
| x2d = x.reshape(-1, orig_shape[-1]) | |
| bias = self.bias.to(dtype=x.dtype) if (self.bias is not None and self.bias.dtype != x.dtype) else self.bias | |
| out = ck.w4a8_int8_linear( | |
| x2d, | |
| self.qdata, | |
| self.s_rel, | |
| self.s_channel, | |
| codebook=self.codebook, | |
| bias=bias, | |
| group_size=self.group_size, | |
| convrot_groupsize=self.convrot_groupsize, | |
| out_dtype=x.dtype, | |
| ) | |
| return out.reshape(*orig_shape[:-1], self.out_features) | |
| def _set_submodule(root: nn.Module, name: str, new_mod: nn.Module) -> None: | |
| parts = name.split(".") | |
| parent = root | |
| for p in parts[:-1]: | |
| parent = getattr(parent, p) | |
| setattr(parent, parts[-1], new_mod) | |
| def apply_convrot_state_dict( | |
| module: nn.Module, | |
| safetensors_path: str, | |
| device: str = "cuda", | |
| dtype: torch.dtype = torch.bfloat16, | |
| ) -> dict: | |
| """Load a ConvRot safetensors file into a PyTorch module, replacing quantized nn.Linear layers.""" | |
| sd = load_file(safetensors_path, device="cpu") | |
| quant_prefixes = sorted(k[:-len(".comfy_quant")] for k in sd.keys() if k.endswith(".comfy_quant")) | |
| quant_set = set(quant_prefixes) | |
| # 1. Load all non-quantized parameters/buffers first | |
| regular_sd = {} | |
| for k, v in sd.items(): | |
| if k.endswith((".comfy_quant", ".weight_scale", ".weight_s_rel", ".weight_s_channel", ".weight_codebook")): | |
| continue | |
| if k.endswith(".weight") and k[:-7] in quant_set: | |
| continue | |
| if k.endswith(".bias") and k[:-5] in quant_set: | |
| continue | |
| regular_sd[k] = v | |
| missing, unexpected = module.load_state_dict(regular_sd, strict=False, assign=True) | |
| # 2. Replace each quantized Linear with ConvRotInt8Linear or ConvRotPackedLinear | |
| counts = {"int8": 0, "w6a8_int8": 0, "asym_w4a8_int8": 0} | |
| for prefix in quant_prefixes: | |
| conf = _decode_comfy_quant(sd[f"{prefix}.comfy_quant"]) | |
| fmt = conf["format"] | |
| bias = sd.get(f"{prefix}.bias", None) | |
| if bias is not None: | |
| bias = bias.to(device=device, dtype=dtype) | |
| if fmt == "int8_tensorwise": | |
| layer = ConvRotInt8Linear( | |
| weight=sd[f"{prefix}.weight"].to(device=device), | |
| weight_scale=sd[f"{prefix}.weight_scale"].to(device=device), | |
| bias=bias, | |
| convrot=conf.get("convrot", True), | |
| convrot_groupsize=conf.get("convrot_groupsize", 256), | |
| ) | |
| counts["int8"] += 1 | |
| elif fmt in ("w6a8_int8", "asym_w4a8_int8"): | |
| cb = sd.get(f"{prefix}.weight_codebook", None) | |
| layer = ConvRotPackedLinear( | |
| qdata=sd[f"{prefix}.weight"].to(device=device), | |
| s_rel=sd[f"{prefix}.weight_s_rel"].to(device=device), | |
| s_channel=sd[f"{prefix}.weight_s_channel"].to(device=device), | |
| codebook=cb.to(device=device) if cb is not None else None, | |
| bias=bias, | |
| group_size=conf.get("group_size", 16), | |
| convrot_groupsize=conf.get("convrot_groupsize", 256), | |
| ) | |
| counts[fmt] += 1 | |
| else: | |
| raise ValueError(f"Unsupported comfy_quant format: {fmt}") | |
| _set_submodule(module, prefix, layer) | |
| # Materialize any non-persistent rotary_emb buffers still on meta | |
| for m_name, mod in list(module.named_modules()): | |
| if any(b.device.type == "meta" for b in mod.buffers(recurse=False)): | |
| if hasattr(mod, "config"): | |
| _set_submodule(module, m_name, mod.__class__(config=mod.config).to(device)) | |
| module.to(device=device) | |
| del sd, regular_sd | |
| return {"counts": counts, "missing_nonquant": [m for m in missing if m[:-7] not in quant_set and m[:-5] not in quant_set]} | |
| def load_glm_image_convrot( | |
| base_model: str, | |
| transformer_path: Optional[str] = None, | |
| vlm_path: Optional[str] = None, | |
| device: str = "cuda", | |
| dtype: torch.dtype = torch.bfloat16, | |
| ) -> tuple[GlmImagePipeline, dict]: | |
| """Load GlmImagePipeline with ConvRot quantized transformer and/or vision_language_encoder.""" | |
| t0 = time.perf_counter() | |
| from diffusers import GlmImageTransformer2DModel | |
| from transformers import GlmImageForConditionalGeneration, AutoConfig | |
| kwargs = {"dtype": dtype} | |
| info = {} | |
| if transformer_path is not None: | |
| tcfg_path = os.path.join(base_model, "transformer", "config.json") | |
| tcfg = json.load(open(tcfg_path, "r", encoding="utf-8")) | |
| with torch.device("meta"): | |
| transformer = GlmImageTransformer2DModel.from_config(tcfg) | |
| info["transformer"] = apply_convrot_state_dict(transformer, transformer_path, device=device, dtype=dtype) | |
| kwargs["transformer"] = transformer | |
| if vlm_path is not None: | |
| vcfg = AutoConfig.from_pretrained(os.path.join(base_model, "vision_language_encoder")) | |
| with torch.device("meta"): | |
| vlm = GlmImageForConditionalGeneration(vcfg) | |
| info["vision_language_encoder"] = apply_convrot_state_dict(vlm, vlm_path, device=device, dtype=dtype) | |
| kwargs["vision_language_encoder"] = vlm | |
| pipe = GlmImagePipeline.from_pretrained(base_model, **kwargs) | |
| pipe.to(device) | |
| torch.cuda.synchronize() | |
| info["load_s"] = round(time.perf_counter() - t0, 3) | |
| return pipe, info | |