GLM-Image-ConvRot / glm_image_convrot.py
SUSSYEATING's picture
Release GLM-Image ConvRot INT8, INT6 & INT4 checkpoints, metrics, and showcases
9ca4e71
Raw History Blame Contribute Delete
8.55 kB
"""Fast ConvRot INT8 / W6A8 / W4A8 runtime loader for zai-org/GLM-Image using comfy-kitchen."""
from __future__ import annotations
import json
import os
import time
from typing import Optional
import torch
import torch.nn as nn
from safetensors import safe_open
from safetensors.torch import load_file
import comfy_kitchen as ck
from diffusers import GlmImagePipeline
def _decode_comfy_quant(tensor: torch.Tensor) -> dict:
return json.loads(bytes(tensor.cpu().numpy().tolist()).decode("utf-8"))
class ConvRotInt8Linear(nn.Module):
"""Direct comfy-kitchen ConvRot INT8 linear layer (bypasses Tensor subclass dispatch overhead)."""
def __init__(
self,
weight: torch.Tensor,
weight_scale: torch.Tensor,
bias: Optional[torch.Tensor] = None,
convrot: bool = True,
convrot_groupsize: int = 256,
):
super().__init__()
self.register_buffer("weight", weight.contiguous())
self.register_buffer("weight_scale", weight_scale.to(torch.float32).contiguous())
if bias is not None:
self.register_buffer("bias", bias.contiguous())
else:
self.bias = None
self.convrot = bool(convrot)
self.convrot_groupsize = int(convrot_groupsize)
self.out_features, self.in_features = int(weight.shape[0]), int(weight.shape[1])
def forward(self, x: torch.Tensor) -> torch.Tensor:
orig_shape = x.shape
x2d = x.reshape(-1, orig_shape[-1])
bias = self.bias.to(dtype=x.dtype) if (self.bias is not None and self.bias.dtype != x.dtype) else self.bias
out = ck.int8_linear(
x2d,
self.weight,
self.weight_scale,
bias=bias,
out_dtype=x.dtype,
convrot=self.convrot,
convrot_groupsize=self.convrot_groupsize,
)
return out.reshape(*orig_shape[:-1], self.out_features)
class ConvRotPackedLinear(nn.Module):
"""Direct comfy-kitchen ConvRot W6A8 (w6a8_int8) / W4A8 (asym_w4a8_int8) linear layer."""
def __init__(
self,
qdata: torch.Tensor,
s_rel: torch.Tensor,
s_channel: torch.Tensor,
codebook: Optional[torch.Tensor] = None,
bias: Optional[torch.Tensor] = None,
group_size: int = 16,
convrot_groupsize: int = 256,
):
super().__init__()
self.register_buffer("qdata", qdata.contiguous())
self.register_buffer("s_rel", s_rel.contiguous())
self.register_buffer("s_channel", s_channel.to(torch.float32).contiguous())
if codebook is not None:
self.register_buffer("codebook", codebook.to(torch.float32).contiguous())
else:
self.codebook = None
if bias is not None:
self.register_buffer("bias", bias.contiguous())
else:
self.bias = None
self.group_size = int(group_size)
self.convrot_groupsize = int(convrot_groupsize)
self.out_features = int(s_channel.shape[0])
self.in_features = int(s_rel.shape[1]) * self.group_size
def forward(self, x: torch.Tensor) -> torch.Tensor:
orig_shape = x.shape
x2d = x.reshape(-1, orig_shape[-1])
bias = self.bias.to(dtype=x.dtype) if (self.bias is not None and self.bias.dtype != x.dtype) else self.bias
out = ck.w4a8_int8_linear(
x2d,
self.qdata,
self.s_rel,
self.s_channel,
codebook=self.codebook,
bias=bias,
group_size=self.group_size,
convrot_groupsize=self.convrot_groupsize,
out_dtype=x.dtype,
)
return out.reshape(*orig_shape[:-1], self.out_features)
def _set_submodule(root: nn.Module, name: str, new_mod: nn.Module) -> None:
parts = name.split(".")
parent = root
for p in parts[:-1]:
parent = getattr(parent, p)
setattr(parent, parts[-1], new_mod)
def apply_convrot_state_dict(
module: nn.Module,
safetensors_path: str,
device: str = "cuda",
dtype: torch.dtype = torch.bfloat16,
) -> dict:
"""Load a ConvRot safetensors file into a PyTorch module, replacing quantized nn.Linear layers."""
sd = load_file(safetensors_path, device="cpu")
quant_prefixes = sorted(k[:-len(".comfy_quant")] for k in sd.keys() if k.endswith(".comfy_quant"))
quant_set = set(quant_prefixes)
# 1. Load all non-quantized parameters/buffers first
regular_sd = {}
for k, v in sd.items():
if k.endswith((".comfy_quant", ".weight_scale", ".weight_s_rel", ".weight_s_channel", ".weight_codebook")):
continue
if k.endswith(".weight") and k[:-7] in quant_set:
continue
if k.endswith(".bias") and k[:-5] in quant_set:
continue
regular_sd[k] = v
missing, unexpected = module.load_state_dict(regular_sd, strict=False, assign=True)
# 2. Replace each quantized Linear with ConvRotInt8Linear or ConvRotPackedLinear
counts = {"int8": 0, "w6a8_int8": 0, "asym_w4a8_int8": 0}
for prefix in quant_prefixes:
conf = _decode_comfy_quant(sd[f"{prefix}.comfy_quant"])
fmt = conf["format"]
bias = sd.get(f"{prefix}.bias", None)
if bias is not None:
bias = bias.to(device=device, dtype=dtype)
if fmt == "int8_tensorwise":
layer = ConvRotInt8Linear(
weight=sd[f"{prefix}.weight"].to(device=device),
weight_scale=sd[f"{prefix}.weight_scale"].to(device=device),
bias=bias,
convrot=conf.get("convrot", True),
convrot_groupsize=conf.get("convrot_groupsize", 256),
)
counts["int8"] += 1
elif fmt in ("w6a8_int8", "asym_w4a8_int8"):
cb = sd.get(f"{prefix}.weight_codebook", None)
layer = ConvRotPackedLinear(
qdata=sd[f"{prefix}.weight"].to(device=device),
s_rel=sd[f"{prefix}.weight_s_rel"].to(device=device),
s_channel=sd[f"{prefix}.weight_s_channel"].to(device=device),
codebook=cb.to(device=device) if cb is not None else None,
bias=bias,
group_size=conf.get("group_size", 16),
convrot_groupsize=conf.get("convrot_groupsize", 256),
)
counts[fmt] += 1
else:
raise ValueError(f"Unsupported comfy_quant format: {fmt}")
_set_submodule(module, prefix, layer)
# Materialize any non-persistent rotary_emb buffers still on meta
for m_name, mod in list(module.named_modules()):
if any(b.device.type == "meta" for b in mod.buffers(recurse=False)):
if hasattr(mod, "config"):
_set_submodule(module, m_name, mod.__class__(config=mod.config).to(device))
module.to(device=device)
del sd, regular_sd
return {"counts": counts, "missing_nonquant": [m for m in missing if m[:-7] not in quant_set and m[:-5] not in quant_set]}
def load_glm_image_convrot(
base_model: str,
transformer_path: Optional[str] = None,
vlm_path: Optional[str] = None,
device: str = "cuda",
dtype: torch.dtype = torch.bfloat16,
) -> tuple[GlmImagePipeline, dict]:
"""Load GlmImagePipeline with ConvRot quantized transformer and/or vision_language_encoder."""
t0 = time.perf_counter()
from diffusers import GlmImageTransformer2DModel
from transformers import GlmImageForConditionalGeneration, AutoConfig
kwargs = {"dtype": dtype}
info = {}
if transformer_path is not None:
tcfg_path = os.path.join(base_model, "transformer", "config.json")
tcfg = json.load(open(tcfg_path, "r", encoding="utf-8"))
with torch.device("meta"):
transformer = GlmImageTransformer2DModel.from_config(tcfg)
info["transformer"] = apply_convrot_state_dict(transformer, transformer_path, device=device, dtype=dtype)
kwargs["transformer"] = transformer
if vlm_path is not None:
vcfg = AutoConfig.from_pretrained(os.path.join(base_model, "vision_language_encoder"))
with torch.device("meta"):
vlm = GlmImageForConditionalGeneration(vcfg)
info["vision_language_encoder"] = apply_convrot_state_dict(vlm, vlm_path, device=device, dtype=dtype)
kwargs["vision_language_encoder"] = vlm
pipe = GlmImagePipeline.from_pretrained(base_model, **kwargs)
pipe.to(device)
torch.cuda.synchronize()
info["load_s"] = round(time.perf_counter() - t0, 3)
return pipe, info