File size: 8,550 Bytes
9ca4e71
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
"""Fast ConvRot INT8 / W6A8 / W4A8 runtime loader for zai-org/GLM-Image using comfy-kitchen."""
from __future__ import annotations

import json
import os
import time
from typing import Optional

import torch
import torch.nn as nn
from safetensors import safe_open
from safetensors.torch import load_file

import comfy_kitchen as ck
from diffusers import GlmImagePipeline


def _decode_comfy_quant(tensor: torch.Tensor) -> dict:
    return json.loads(bytes(tensor.cpu().numpy().tolist()).decode("utf-8"))


class ConvRotInt8Linear(nn.Module):
    """Direct comfy-kitchen ConvRot INT8 linear layer (bypasses Tensor subclass dispatch overhead)."""

    def __init__(
        self,
        weight: torch.Tensor,
        weight_scale: torch.Tensor,
        bias: Optional[torch.Tensor] = None,
        convrot: bool = True,
        convrot_groupsize: int = 256,
    ):
        super().__init__()
        self.register_buffer("weight", weight.contiguous())
        self.register_buffer("weight_scale", weight_scale.to(torch.float32).contiguous())
        if bias is not None:
            self.register_buffer("bias", bias.contiguous())
        else:
            self.bias = None
        self.convrot = bool(convrot)
        self.convrot_groupsize = int(convrot_groupsize)
        self.out_features, self.in_features = int(weight.shape[0]), int(weight.shape[1])

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        orig_shape = x.shape
        x2d = x.reshape(-1, orig_shape[-1])
        bias = self.bias.to(dtype=x.dtype) if (self.bias is not None and self.bias.dtype != x.dtype) else self.bias
        out = ck.int8_linear(
            x2d,
            self.weight,
            self.weight_scale,
            bias=bias,
            out_dtype=x.dtype,
            convrot=self.convrot,
            convrot_groupsize=self.convrot_groupsize,
        )
        return out.reshape(*orig_shape[:-1], self.out_features)


class ConvRotPackedLinear(nn.Module):
    """Direct comfy-kitchen ConvRot W6A8 (w6a8_int8) / W4A8 (asym_w4a8_int8) linear layer."""

    def __init__(
        self,
        qdata: torch.Tensor,
        s_rel: torch.Tensor,
        s_channel: torch.Tensor,
        codebook: Optional[torch.Tensor] = None,
        bias: Optional[torch.Tensor] = None,
        group_size: int = 16,
        convrot_groupsize: int = 256,
    ):
        super().__init__()
        self.register_buffer("qdata", qdata.contiguous())
        self.register_buffer("s_rel", s_rel.contiguous())
        self.register_buffer("s_channel", s_channel.to(torch.float32).contiguous())
        if codebook is not None:
            self.register_buffer("codebook", codebook.to(torch.float32).contiguous())
        else:
            self.codebook = None
        if bias is not None:
            self.register_buffer("bias", bias.contiguous())
        else:
            self.bias = None
        self.group_size = int(group_size)
        self.convrot_groupsize = int(convrot_groupsize)
        self.out_features = int(s_channel.shape[0])
        self.in_features = int(s_rel.shape[1]) * self.group_size

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        orig_shape = x.shape
        x2d = x.reshape(-1, orig_shape[-1])
        bias = self.bias.to(dtype=x.dtype) if (self.bias is not None and self.bias.dtype != x.dtype) else self.bias
        out = ck.w4a8_int8_linear(
            x2d,
            self.qdata,
            self.s_rel,
            self.s_channel,
            codebook=self.codebook,
            bias=bias,
            group_size=self.group_size,
            convrot_groupsize=self.convrot_groupsize,
            out_dtype=x.dtype,
        )
        return out.reshape(*orig_shape[:-1], self.out_features)


def _set_submodule(root: nn.Module, name: str, new_mod: nn.Module) -> None:
    parts = name.split(".")
    parent = root
    for p in parts[:-1]:
        parent = getattr(parent, p)
    setattr(parent, parts[-1], new_mod)


def apply_convrot_state_dict(
    module: nn.Module,
    safetensors_path: str,
    device: str = "cuda",
    dtype: torch.dtype = torch.bfloat16,
) -> dict:
    """Load a ConvRot safetensors file into a PyTorch module, replacing quantized nn.Linear layers."""
    sd = load_file(safetensors_path, device="cpu")
    quant_prefixes = sorted(k[:-len(".comfy_quant")] for k in sd.keys() if k.endswith(".comfy_quant"))
    quant_set = set(quant_prefixes)

    # 1. Load all non-quantized parameters/buffers first
    regular_sd = {}
    for k, v in sd.items():
        if k.endswith((".comfy_quant", ".weight_scale", ".weight_s_rel", ".weight_s_channel", ".weight_codebook")):
            continue
        if k.endswith(".weight") and k[:-7] in quant_set:
            continue
        if k.endswith(".bias") and k[:-5] in quant_set:
            continue
        regular_sd[k] = v

    missing, unexpected = module.load_state_dict(regular_sd, strict=False, assign=True)
    # 2. Replace each quantized Linear with ConvRotInt8Linear or ConvRotPackedLinear
    counts = {"int8": 0, "w6a8_int8": 0, "asym_w4a8_int8": 0}
    for prefix in quant_prefixes:
        conf = _decode_comfy_quant(sd[f"{prefix}.comfy_quant"])
        fmt = conf["format"]
        bias = sd.get(f"{prefix}.bias", None)
        if bias is not None:
            bias = bias.to(device=device, dtype=dtype)
        if fmt == "int8_tensorwise":
            layer = ConvRotInt8Linear(
                weight=sd[f"{prefix}.weight"].to(device=device),
                weight_scale=sd[f"{prefix}.weight_scale"].to(device=device),
                bias=bias,
                convrot=conf.get("convrot", True),
                convrot_groupsize=conf.get("convrot_groupsize", 256),
            )
            counts["int8"] += 1
        elif fmt in ("w6a8_int8", "asym_w4a8_int8"):
            cb = sd.get(f"{prefix}.weight_codebook", None)
            layer = ConvRotPackedLinear(
                qdata=sd[f"{prefix}.weight"].to(device=device),
                s_rel=sd[f"{prefix}.weight_s_rel"].to(device=device),
                s_channel=sd[f"{prefix}.weight_s_channel"].to(device=device),
                codebook=cb.to(device=device) if cb is not None else None,
                bias=bias,
                group_size=conf.get("group_size", 16),
                convrot_groupsize=conf.get("convrot_groupsize", 256),
            )
            counts[fmt] += 1
        else:
            raise ValueError(f"Unsupported comfy_quant format: {fmt}")
        _set_submodule(module, prefix, layer)

    # Materialize any non-persistent rotary_emb buffers still on meta
    for m_name, mod in list(module.named_modules()):
        if any(b.device.type == "meta" for b in mod.buffers(recurse=False)):
            if hasattr(mod, "config"):
                _set_submodule(module, m_name, mod.__class__(config=mod.config).to(device))

    module.to(device=device)
    del sd, regular_sd
    return {"counts": counts, "missing_nonquant": [m for m in missing if m[:-7] not in quant_set and m[:-5] not in quant_set]}


def load_glm_image_convrot(
    base_model: str,
    transformer_path: Optional[str] = None,
    vlm_path: Optional[str] = None,
    device: str = "cuda",
    dtype: torch.dtype = torch.bfloat16,
) -> tuple[GlmImagePipeline, dict]:
    """Load GlmImagePipeline with ConvRot quantized transformer and/or vision_language_encoder."""
    t0 = time.perf_counter()
    from diffusers import GlmImageTransformer2DModel
    from transformers import GlmImageForConditionalGeneration, AutoConfig

    kwargs = {"dtype": dtype}
    info = {}

    if transformer_path is not None:
        tcfg_path = os.path.join(base_model, "transformer", "config.json")
        tcfg = json.load(open(tcfg_path, "r", encoding="utf-8"))
        with torch.device("meta"):
            transformer = GlmImageTransformer2DModel.from_config(tcfg)
        info["transformer"] = apply_convrot_state_dict(transformer, transformer_path, device=device, dtype=dtype)
        kwargs["transformer"] = transformer

    if vlm_path is not None:
        vcfg = AutoConfig.from_pretrained(os.path.join(base_model, "vision_language_encoder"))
        with torch.device("meta"):
            vlm = GlmImageForConditionalGeneration(vcfg)
        info["vision_language_encoder"] = apply_convrot_state_dict(vlm, vlm_path, device=device, dtype=dtype)
        kwargs["vision_language_encoder"] = vlm

    pipe = GlmImagePipeline.from_pretrained(base_model, **kwargs)
    pipe.to(device)
    torch.cuda.synchronize()
    info["load_s"] = round(time.perf_counter() - t0, 3)
    return pipe, info