Instructions to use SUSSYEATING/GLM-Image-ConvRot with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Diffusers
How to use SUSSYEATING/GLM-Image-ConvRot with Diffusers:
pip install -U diffusers transformers accelerate
import torch from diffusers import DiffusionPipeline # switch to "mps" for apple devices pipe = DiffusionPipeline.from_pretrained("SUSSYEATING/GLM-Image-ConvRot", dtype=torch.bfloat16, device_map="cuda") prompt = "Astronaut in a jungle, cold color palette, muted colors, detailed, 8k" image = pipe(prompt).images[0] - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- Draw Things
- DiffusionBee
File size: 8,550 Bytes
9ca4e71 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 | """Fast ConvRot INT8 / W6A8 / W4A8 runtime loader for zai-org/GLM-Image using comfy-kitchen."""
from __future__ import annotations
import json
import os
import time
from typing import Optional
import torch
import torch.nn as nn
from safetensors import safe_open
from safetensors.torch import load_file
import comfy_kitchen as ck
from diffusers import GlmImagePipeline
def _decode_comfy_quant(tensor: torch.Tensor) -> dict:
return json.loads(bytes(tensor.cpu().numpy().tolist()).decode("utf-8"))
class ConvRotInt8Linear(nn.Module):
"""Direct comfy-kitchen ConvRot INT8 linear layer (bypasses Tensor subclass dispatch overhead)."""
def __init__(
self,
weight: torch.Tensor,
weight_scale: torch.Tensor,
bias: Optional[torch.Tensor] = None,
convrot: bool = True,
convrot_groupsize: int = 256,
):
super().__init__()
self.register_buffer("weight", weight.contiguous())
self.register_buffer("weight_scale", weight_scale.to(torch.float32).contiguous())
if bias is not None:
self.register_buffer("bias", bias.contiguous())
else:
self.bias = None
self.convrot = bool(convrot)
self.convrot_groupsize = int(convrot_groupsize)
self.out_features, self.in_features = int(weight.shape[0]), int(weight.shape[1])
def forward(self, x: torch.Tensor) -> torch.Tensor:
orig_shape = x.shape
x2d = x.reshape(-1, orig_shape[-1])
bias = self.bias.to(dtype=x.dtype) if (self.bias is not None and self.bias.dtype != x.dtype) else self.bias
out = ck.int8_linear(
x2d,
self.weight,
self.weight_scale,
bias=bias,
out_dtype=x.dtype,
convrot=self.convrot,
convrot_groupsize=self.convrot_groupsize,
)
return out.reshape(*orig_shape[:-1], self.out_features)
class ConvRotPackedLinear(nn.Module):
"""Direct comfy-kitchen ConvRot W6A8 (w6a8_int8) / W4A8 (asym_w4a8_int8) linear layer."""
def __init__(
self,
qdata: torch.Tensor,
s_rel: torch.Tensor,
s_channel: torch.Tensor,
codebook: Optional[torch.Tensor] = None,
bias: Optional[torch.Tensor] = None,
group_size: int = 16,
convrot_groupsize: int = 256,
):
super().__init__()
self.register_buffer("qdata", qdata.contiguous())
self.register_buffer("s_rel", s_rel.contiguous())
self.register_buffer("s_channel", s_channel.to(torch.float32).contiguous())
if codebook is not None:
self.register_buffer("codebook", codebook.to(torch.float32).contiguous())
else:
self.codebook = None
if bias is not None:
self.register_buffer("bias", bias.contiguous())
else:
self.bias = None
self.group_size = int(group_size)
self.convrot_groupsize = int(convrot_groupsize)
self.out_features = int(s_channel.shape[0])
self.in_features = int(s_rel.shape[1]) * self.group_size
def forward(self, x: torch.Tensor) -> torch.Tensor:
orig_shape = x.shape
x2d = x.reshape(-1, orig_shape[-1])
bias = self.bias.to(dtype=x.dtype) if (self.bias is not None and self.bias.dtype != x.dtype) else self.bias
out = ck.w4a8_int8_linear(
x2d,
self.qdata,
self.s_rel,
self.s_channel,
codebook=self.codebook,
bias=bias,
group_size=self.group_size,
convrot_groupsize=self.convrot_groupsize,
out_dtype=x.dtype,
)
return out.reshape(*orig_shape[:-1], self.out_features)
def _set_submodule(root: nn.Module, name: str, new_mod: nn.Module) -> None:
parts = name.split(".")
parent = root
for p in parts[:-1]:
parent = getattr(parent, p)
setattr(parent, parts[-1], new_mod)
def apply_convrot_state_dict(
module: nn.Module,
safetensors_path: str,
device: str = "cuda",
dtype: torch.dtype = torch.bfloat16,
) -> dict:
"""Load a ConvRot safetensors file into a PyTorch module, replacing quantized nn.Linear layers."""
sd = load_file(safetensors_path, device="cpu")
quant_prefixes = sorted(k[:-len(".comfy_quant")] for k in sd.keys() if k.endswith(".comfy_quant"))
quant_set = set(quant_prefixes)
# 1. Load all non-quantized parameters/buffers first
regular_sd = {}
for k, v in sd.items():
if k.endswith((".comfy_quant", ".weight_scale", ".weight_s_rel", ".weight_s_channel", ".weight_codebook")):
continue
if k.endswith(".weight") and k[:-7] in quant_set:
continue
if k.endswith(".bias") and k[:-5] in quant_set:
continue
regular_sd[k] = v
missing, unexpected = module.load_state_dict(regular_sd, strict=False, assign=True)
# 2. Replace each quantized Linear with ConvRotInt8Linear or ConvRotPackedLinear
counts = {"int8": 0, "w6a8_int8": 0, "asym_w4a8_int8": 0}
for prefix in quant_prefixes:
conf = _decode_comfy_quant(sd[f"{prefix}.comfy_quant"])
fmt = conf["format"]
bias = sd.get(f"{prefix}.bias", None)
if bias is not None:
bias = bias.to(device=device, dtype=dtype)
if fmt == "int8_tensorwise":
layer = ConvRotInt8Linear(
weight=sd[f"{prefix}.weight"].to(device=device),
weight_scale=sd[f"{prefix}.weight_scale"].to(device=device),
bias=bias,
convrot=conf.get("convrot", True),
convrot_groupsize=conf.get("convrot_groupsize", 256),
)
counts["int8"] += 1
elif fmt in ("w6a8_int8", "asym_w4a8_int8"):
cb = sd.get(f"{prefix}.weight_codebook", None)
layer = ConvRotPackedLinear(
qdata=sd[f"{prefix}.weight"].to(device=device),
s_rel=sd[f"{prefix}.weight_s_rel"].to(device=device),
s_channel=sd[f"{prefix}.weight_s_channel"].to(device=device),
codebook=cb.to(device=device) if cb is not None else None,
bias=bias,
group_size=conf.get("group_size", 16),
convrot_groupsize=conf.get("convrot_groupsize", 256),
)
counts[fmt] += 1
else:
raise ValueError(f"Unsupported comfy_quant format: {fmt}")
_set_submodule(module, prefix, layer)
# Materialize any non-persistent rotary_emb buffers still on meta
for m_name, mod in list(module.named_modules()):
if any(b.device.type == "meta" for b in mod.buffers(recurse=False)):
if hasattr(mod, "config"):
_set_submodule(module, m_name, mod.__class__(config=mod.config).to(device))
module.to(device=device)
del sd, regular_sd
return {"counts": counts, "missing_nonquant": [m for m in missing if m[:-7] not in quant_set and m[:-5] not in quant_set]}
def load_glm_image_convrot(
base_model: str,
transformer_path: Optional[str] = None,
vlm_path: Optional[str] = None,
device: str = "cuda",
dtype: torch.dtype = torch.bfloat16,
) -> tuple[GlmImagePipeline, dict]:
"""Load GlmImagePipeline with ConvRot quantized transformer and/or vision_language_encoder."""
t0 = time.perf_counter()
from diffusers import GlmImageTransformer2DModel
from transformers import GlmImageForConditionalGeneration, AutoConfig
kwargs = {"dtype": dtype}
info = {}
if transformer_path is not None:
tcfg_path = os.path.join(base_model, "transformer", "config.json")
tcfg = json.load(open(tcfg_path, "r", encoding="utf-8"))
with torch.device("meta"):
transformer = GlmImageTransformer2DModel.from_config(tcfg)
info["transformer"] = apply_convrot_state_dict(transformer, transformer_path, device=device, dtype=dtype)
kwargs["transformer"] = transformer
if vlm_path is not None:
vcfg = AutoConfig.from_pretrained(os.path.join(base_model, "vision_language_encoder"))
with torch.device("meta"):
vlm = GlmImageForConditionalGeneration(vcfg)
info["vision_language_encoder"] = apply_convrot_state_dict(vlm, vlm_path, device=device, dtype=dtype)
kwargs["vision_language_encoder"] = vlm
pipe = GlmImagePipeline.from_pretrained(base_model, **kwargs)
pipe.to(device)
torch.cuda.synchronize()
info["load_s"] = round(time.perf_counter() - t0, 3)
return pipe, info
|