Instructions to use SUSSYEATING/GLM-Image-ConvRot with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Diffusers
How to use SUSSYEATING/GLM-Image-ConvRot with Diffusers:
pip install -U diffusers transformers accelerate
import torch from diffusers import DiffusionPipeline # switch to "mps" for apple devices pipe = DiffusionPipeline.from_pretrained("SUSSYEATING/GLM-Image-ConvRot", dtype=torch.bfloat16, device_map="cuda") prompt = "Astronaut in a jungle, cold color palette, muted colors, detailed, 8k" image = pipe(prompt).images[0] - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- Draw Things
- DiffusionBee
Commit ·
9ca4e71
0
Parent(s):
Release GLM-Image ConvRot INT8, INT6 & INT4 checkpoints, metrics, and showcases
Browse files- .gitattributes +39 -0
- README.md +105 -0
- comparisons/t2i_cabin_s42_4way.png +3 -0
- comparisons/t2i_cabin_s42_dit_4way.png +3 -0
- comparisons/t2i_poster_s1337_4way.png +3 -0
- comparisons/t2i_watch_s777_4way.png +3 -0
- glm_image_convrot.py +217 -0
- int4/glm-image-transformer-int4-convrot.safetensors +3 -0
- int4/glm-image-vlm-int4-convrot.safetensors +3 -0
- int6/glm-image-transformer-int6-convrot.safetensors +3 -0
- int6/glm-image-vlm-int6-convrot.safetensors +3 -0
- int8/glm-image-transformer-int8-convrot.safetensors +3 -0
- int8/glm-image-vlm-int8-convrot.safetensors +3 -0
- metrics/g4_glm_bf16_stats.json +25 -0
- metrics/g4_glm_int4_stats.json +78 -0
- metrics/g4_glm_int6_stats.json +78 -0
- metrics/g4_glm_int8_stats.json +78 -0
- metrics/g4_glm_vlm_teacher_forced.json +29 -0
- scripts/eval_glm_image.py +161 -0
.gitattributes
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
*.7z filter=lfs diff=lfs merge=lfs -text
|
| 2 |
+
*.arrow filter=lfs diff=lfs merge=lfs -text
|
| 3 |
+
*.bin filter=lfs diff=lfs merge=lfs -text
|
| 4 |
+
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
| 5 |
+
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
| 6 |
+
*.ftz filter=lfs diff=lfs merge=lfs -text
|
| 7 |
+
*.gz filter=lfs diff=lfs merge=lfs -text
|
| 8 |
+
*.h5 filter=lfs diff=lfs merge=lfs -text
|
| 9 |
+
*.joblib filter=lfs diff=lfs merge=lfs -text
|
| 10 |
+
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
| 11 |
+
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
| 12 |
+
*.model filter=lfs diff=lfs merge=lfs -text
|
| 13 |
+
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
| 14 |
+
*.npy filter=lfs diff=lfs merge=lfs -text
|
| 15 |
+
*.npz filter=lfs diff=lfs merge=lfs -text
|
| 16 |
+
*.onnx filter=lfs diff=lfs merge=lfs -text
|
| 17 |
+
*.ot filter=lfs diff=lfs merge=lfs -text
|
| 18 |
+
*.parquet filter=lfs diff=lfs merge=lfs -text
|
| 19 |
+
*.pb filter=lfs diff=lfs merge=lfs -text
|
| 20 |
+
*.pickle filter=lfs diff=lfs merge=lfs -text
|
| 21 |
+
*.pkl filter=lfs diff=lfs merge=lfs -text
|
| 22 |
+
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 23 |
+
*.pth filter=lfs diff=lfs merge=lfs -text
|
| 24 |
+
*.rar filter=lfs diff=lfs merge=lfs -text
|
| 25 |
+
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
| 26 |
+
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
| 27 |
+
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
| 28 |
+
*.tar filter=lfs diff=lfs merge=lfs -text
|
| 29 |
+
*.tflite filter=lfs diff=lfs merge=lfs -text
|
| 30 |
+
*.tgz filter=lfs diff=lfs merge=lfs -text
|
| 31 |
+
*.wasm filter=lfs diff=lfs merge=lfs -text
|
| 32 |
+
*.xz filter=lfs diff=lfs merge=lfs -text
|
| 33 |
+
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
+
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
+
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
comparisons/t2i_cabin_s42_4way.png filter=lfs diff=lfs merge=lfs -text
|
| 37 |
+
comparisons/t2i_cabin_s42_dit_4way.png filter=lfs diff=lfs merge=lfs -text
|
| 38 |
+
comparisons/t2i_poster_s1337_4way.png filter=lfs diff=lfs merge=lfs -text
|
| 39 |
+
comparisons/t2i_watch_s777_4way.png filter=lfs diff=lfs merge=lfs -text
|
README.md
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: mit
|
| 3 |
+
language:
|
| 4 |
+
- zh
|
| 5 |
+
- en
|
| 6 |
+
library_name: diffusers
|
| 7 |
+
pipeline_tag: text-to-image
|
| 8 |
+
base_model: zai-org/GLM-Image
|
| 9 |
+
tags:
|
| 10 |
+
- glm-image
|
| 11 |
+
- convrot
|
| 12 |
+
- quantization
|
| 13 |
+
- int8
|
| 14 |
+
- int6
|
| 15 |
+
- int4
|
| 16 |
+
- comfy-kitchen
|
| 17 |
+
---
|
| 18 |
+
|
| 19 |
+
# GLM-Image ConvRot (`INT8`, `INT6 w6w8`, `INT4 w4w8`)
|
| 20 |
+
|
| 21 |
+
High-fidelity **ConvRot** quantized checkpoints for [**zai-org/GLM-Image**](https://huggingface.co/zai-org/GLM-Image) (hybrid **9B Autoregressive VLM** + **7B Diffusion Decoder DiT**), powered by [`comfy-kitchen`](https://github.com/Comfy-Org/comfy-kitchen) INT8 / W6A8 / W4A8 Tensor Core kernels.
|
| 22 |
+
|
| 23 |
+
- **7B DiT (`transformer`)**: All `180` attention/FFN block linears quantized with ConvRot-256 (`int8_tensorwise`, `w6a8_int8`, `asym_w4a8_int8`) + `30` AdaLN modulation layers (`norm1.linear`) kept at `INT8 ConvRot-256` across all tiers; `prior_token_embedding`, `glyph_projector`, `image_projector`, `time_condition_embed`, `norm_out`, and `proj_out` remain BF16.
|
| 24 |
+
- **9B AR VLM (`vision_language_encoder`)**: All `40` AR LLM layers quantized with SmoothQuant + ConvRot-256 (`q_proj`, `o_proj`, `mlp.gate_up_proj`) and **`INT8 ConvRot-64`** on `mlp.down_proj` (`K = 13696`, divisible by 64) for `w6w8` / `w4w8` compatibility; `40` ViT blocks quantized to `INT8 ConvRot-256`; `k_proj`, `v_proj` (`0.17 GB` total), `embed_tokens`, `lm_head`, and `vqmodel` remain BF16 to preserve KV-cache precision and discrete visual-token accuracy.
|
| 25 |
+
- **Typography & VAE (`text_encoder` ByT5 + `vae`)**: Loaded unquantized directly from `zai-org/GLM-Image` (`1.68 GB` combined) for crisp text rendering and pixel fidelity.
|
| 26 |
+
|
| 27 |
+
---
|
| 28 |
+
|
| 29 |
+
## Verified Benchmarks (`1024x1024`, 50 DiT steps, `guidance_scale=1.5`)
|
| 30 |
+
|
| 31 |
+
| Tier | DiT + VLM Size | Peak VRAM | DiT Warm Time | DiT-Only PSNR (`cabin / poster / watch`) | DiT-Only SSIM | VLM Teacher-Forced Top-1 / KL |
|
| 32 |
+
| :--- | :---: | :---: | :---: | :---: | :---: | :---: |
|
| 33 |
+
| **BF16 Baseline** | `34.08 GB` (`13.85 + 20.23`) | `40.55 GB` | `21.60s` | Ref | `1.0000` | `100.0%` / `0.000` |
|
| 34 |
+
| **INT8 ConvRot** | **`18.10 GB`** (`7.07 + 11.03`, `-46.9%`) | **`24.59 GB`** | **`17.21s`** (**`1.26x`**) | **`47.21 / 42.83 / 38.65 dB`** | **`0.9982 / 0.9965 / 0.9957`** | **`89.71%`** / `0.081` |
|
| 35 |
+
| **INT6 (`w6w8`) ConvRot** | **`15.88 GB`** (`5.94 + 9.94`, `-53.4%`) | **`22.31 GB`** | **`17.95s`** (**`1.20x`**) | **`41.74 / 39.73 / 36.68 dB`** | **`0.9966 / 0.9941 / 0.9933`** | **`89.19%`** / `0.082` |
|
| 36 |
+
| **INT4 (`w4w8`) ConvRot** | **`12.91 GB`** (`4.43 + 8.48`, `-62.1%`) | **`19.40 GB`** | **`17.80s`** (**`1.21x`**) | **`30.44 / 30.55 / 26.37 dB`** | **`0.9723 / 0.9650 / 0.9608`** | **`79.62%`** / `0.316` |
|
| 37 |
+
|
| 38 |
+
> **Note:** In end-to-end sampling, GLM-Image's AR VLM autoregressively samples 4,096 discrete visual tokens, so small logit shifts produce different valid compositions of the same prompt. Both **DiT-isolated** reconstruction (identical BF16 prior tokens -> quantized DiT) and **VLM teacher-forced** visual-token agreement are reported above alongside full end-to-end visual comparisons below.
|
| 39 |
+
|
| 40 |
+
---
|
| 41 |
+
|
| 42 |
+
## Visual Comparisons (`BF16 | INT8 | INT6 w6w8 | INT4 w4w8`)
|
| 43 |
+
|
| 44 |
+
### End-to-End Photorealistic Scene (`t2i_cabin_s42`)
|
| 45 |
+

|
| 46 |
+
|
| 47 |
+
### End-to-End Typography Poster (`t2i_poster_s1337`)
|
| 48 |
+

|
| 49 |
+
|
| 50 |
+
### End-to-End Macro Detail (`t2i_watch_s777`)
|
| 51 |
+

|
| 52 |
+
|
| 53 |
+
### DiT-Isolated Reconstruction Fidelity (`t2i_cabin_s42`, Same BF16 Prior Tokens)
|
| 54 |
+

|
| 55 |
+
|
| 56 |
+
---
|
| 57 |
+
|
| 58 |
+
## Quickstart (`diffusers` + `comfy-kitchen`)
|
| 59 |
+
|
| 60 |
+
```bash
|
| 61 |
+
pip install comfy-kitchen>=0.2.41 diffusers transformers safetensors accelerate
|
| 62 |
+
```
|
| 63 |
+
|
| 64 |
+
```python
|
| 65 |
+
import torch
|
| 66 |
+
from huggingface_hub import hf_hub_download, snapshot_download
|
| 67 |
+
from glm_image_convrot import load_glm_image_convrot
|
| 68 |
+
|
| 69 |
+
base_model = snapshot_download("zai-org/GLM-Image")
|
| 70 |
+
tier = "int6" # "int8", "int6", or "int4"
|
| 71 |
+
|
| 72 |
+
transformer_path = hf_hub_download(
|
| 73 |
+
"SUSSYEATING/GLM-Image-ConvRot",
|
| 74 |
+
f"{tier}/glm-image-transformer-{tier}-convrot.safetensors",
|
| 75 |
+
)
|
| 76 |
+
vlm_path = hf_hub_download(
|
| 77 |
+
"SUSSYEATING/GLM-Image-ConvRot",
|
| 78 |
+
f"{tier}/glm-image-vlm-{tier}-convrot.safetensors",
|
| 79 |
+
)
|
| 80 |
+
|
| 81 |
+
pipe, info = load_glm_image_convrot(
|
| 82 |
+
base_model=base_model,
|
| 83 |
+
transformer_path=transformer_path,
|
| 84 |
+
vlm_path=vlm_path,
|
| 85 |
+
device="cuda",
|
| 86 |
+
dtype=torch.bfloat16,
|
| 87 |
+
)
|
| 88 |
+
|
| 89 |
+
image = pipe(
|
| 90 |
+
prompt="A vintage travel poster for 'ALPS EXPRESS' with bold cream typography, a red steam train crossing a stone viaduct over snowy mountains, 'SWITZERLAND 1964'",
|
| 91 |
+
height=1024,
|
| 92 |
+
width=1024,
|
| 93 |
+
num_inference_steps=50,
|
| 94 |
+
guidance_scale=1.5,
|
| 95 |
+
generator=torch.Generator(device="cuda").manual_seed(1337),
|
| 96 |
+
).images[0]
|
| 97 |
+
image.save("glm_convrot_out.png")
|
| 98 |
+
```
|
| 99 |
+
|
| 100 |
+
---
|
| 101 |
+
|
| 102 |
+
## License & Attribution
|
| 103 |
+
|
| 104 |
+
- Derived from [**zai-org/GLM-Image**](https://huggingface.co/zai-org/GLM-Image), released under the **MIT License**.
|
| 105 |
+
- As noted upstream, the incorporated VQ tokenizer and ViT weights from [**X-Omni/X-Omni-En**](https://huggingface.co/X-Omni/X-Omni-En) remain subject to the **Apache License, Version 2.0**.
|
comparisons/t2i_cabin_s42_4way.png
ADDED
|
Git LFS Details
|
comparisons/t2i_cabin_s42_dit_4way.png
ADDED
|
Git LFS Details
|
comparisons/t2i_poster_s1337_4way.png
ADDED
|
Git LFS Details
|
comparisons/t2i_watch_s777_4way.png
ADDED
|
Git LFS Details
|
glm_image_convrot.py
ADDED
|
@@ -0,0 +1,217 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Fast ConvRot INT8 / W6A8 / W4A8 runtime loader for zai-org/GLM-Image using comfy-kitchen."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
import json
|
| 5 |
+
import os
|
| 6 |
+
import time
|
| 7 |
+
from typing import Optional
|
| 8 |
+
|
| 9 |
+
import torch
|
| 10 |
+
import torch.nn as nn
|
| 11 |
+
from safetensors import safe_open
|
| 12 |
+
from safetensors.torch import load_file
|
| 13 |
+
|
| 14 |
+
import comfy_kitchen as ck
|
| 15 |
+
from diffusers import GlmImagePipeline
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def _decode_comfy_quant(tensor: torch.Tensor) -> dict:
|
| 19 |
+
return json.loads(bytes(tensor.cpu().numpy().tolist()).decode("utf-8"))
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
class ConvRotInt8Linear(nn.Module):
|
| 23 |
+
"""Direct comfy-kitchen ConvRot INT8 linear layer (bypasses Tensor subclass dispatch overhead)."""
|
| 24 |
+
|
| 25 |
+
def __init__(
|
| 26 |
+
self,
|
| 27 |
+
weight: torch.Tensor,
|
| 28 |
+
weight_scale: torch.Tensor,
|
| 29 |
+
bias: Optional[torch.Tensor] = None,
|
| 30 |
+
convrot: bool = True,
|
| 31 |
+
convrot_groupsize: int = 256,
|
| 32 |
+
):
|
| 33 |
+
super().__init__()
|
| 34 |
+
self.register_buffer("weight", weight.contiguous())
|
| 35 |
+
self.register_buffer("weight_scale", weight_scale.to(torch.float32).contiguous())
|
| 36 |
+
if bias is not None:
|
| 37 |
+
self.register_buffer("bias", bias.contiguous())
|
| 38 |
+
else:
|
| 39 |
+
self.bias = None
|
| 40 |
+
self.convrot = bool(convrot)
|
| 41 |
+
self.convrot_groupsize = int(convrot_groupsize)
|
| 42 |
+
self.out_features, self.in_features = int(weight.shape[0]), int(weight.shape[1])
|
| 43 |
+
|
| 44 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 45 |
+
orig_shape = x.shape
|
| 46 |
+
x2d = x.reshape(-1, orig_shape[-1])
|
| 47 |
+
bias = self.bias.to(dtype=x.dtype) if (self.bias is not None and self.bias.dtype != x.dtype) else self.bias
|
| 48 |
+
out = ck.int8_linear(
|
| 49 |
+
x2d,
|
| 50 |
+
self.weight,
|
| 51 |
+
self.weight_scale,
|
| 52 |
+
bias=bias,
|
| 53 |
+
out_dtype=x.dtype,
|
| 54 |
+
convrot=self.convrot,
|
| 55 |
+
convrot_groupsize=self.convrot_groupsize,
|
| 56 |
+
)
|
| 57 |
+
return out.reshape(*orig_shape[:-1], self.out_features)
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
class ConvRotPackedLinear(nn.Module):
|
| 61 |
+
"""Direct comfy-kitchen ConvRot W6A8 (w6a8_int8) / W4A8 (asym_w4a8_int8) linear layer."""
|
| 62 |
+
|
| 63 |
+
def __init__(
|
| 64 |
+
self,
|
| 65 |
+
qdata: torch.Tensor,
|
| 66 |
+
s_rel: torch.Tensor,
|
| 67 |
+
s_channel: torch.Tensor,
|
| 68 |
+
codebook: Optional[torch.Tensor] = None,
|
| 69 |
+
bias: Optional[torch.Tensor] = None,
|
| 70 |
+
group_size: int = 16,
|
| 71 |
+
convrot_groupsize: int = 256,
|
| 72 |
+
):
|
| 73 |
+
super().__init__()
|
| 74 |
+
self.register_buffer("qdata", qdata.contiguous())
|
| 75 |
+
self.register_buffer("s_rel", s_rel.contiguous())
|
| 76 |
+
self.register_buffer("s_channel", s_channel.to(torch.float32).contiguous())
|
| 77 |
+
if codebook is not None:
|
| 78 |
+
self.register_buffer("codebook", codebook.to(torch.float32).contiguous())
|
| 79 |
+
else:
|
| 80 |
+
self.codebook = None
|
| 81 |
+
if bias is not None:
|
| 82 |
+
self.register_buffer("bias", bias.contiguous())
|
| 83 |
+
else:
|
| 84 |
+
self.bias = None
|
| 85 |
+
self.group_size = int(group_size)
|
| 86 |
+
self.convrot_groupsize = int(convrot_groupsize)
|
| 87 |
+
self.out_features = int(s_channel.shape[0])
|
| 88 |
+
self.in_features = int(s_rel.shape[1]) * self.group_size
|
| 89 |
+
|
| 90 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 91 |
+
orig_shape = x.shape
|
| 92 |
+
x2d = x.reshape(-1, orig_shape[-1])
|
| 93 |
+
bias = self.bias.to(dtype=x.dtype) if (self.bias is not None and self.bias.dtype != x.dtype) else self.bias
|
| 94 |
+
out = ck.w4a8_int8_linear(
|
| 95 |
+
x2d,
|
| 96 |
+
self.qdata,
|
| 97 |
+
self.s_rel,
|
| 98 |
+
self.s_channel,
|
| 99 |
+
codebook=self.codebook,
|
| 100 |
+
bias=bias,
|
| 101 |
+
group_size=self.group_size,
|
| 102 |
+
convrot_groupsize=self.convrot_groupsize,
|
| 103 |
+
out_dtype=x.dtype,
|
| 104 |
+
)
|
| 105 |
+
return out.reshape(*orig_shape[:-1], self.out_features)
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def _set_submodule(root: nn.Module, name: str, new_mod: nn.Module) -> None:
|
| 109 |
+
parts = name.split(".")
|
| 110 |
+
parent = root
|
| 111 |
+
for p in parts[:-1]:
|
| 112 |
+
parent = getattr(parent, p)
|
| 113 |
+
setattr(parent, parts[-1], new_mod)
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
def apply_convrot_state_dict(
|
| 117 |
+
module: nn.Module,
|
| 118 |
+
safetensors_path: str,
|
| 119 |
+
device: str = "cuda",
|
| 120 |
+
dtype: torch.dtype = torch.bfloat16,
|
| 121 |
+
) -> dict:
|
| 122 |
+
"""Load a ConvRot safetensors file into a PyTorch module, replacing quantized nn.Linear layers."""
|
| 123 |
+
sd = load_file(safetensors_path, device="cpu")
|
| 124 |
+
quant_prefixes = sorted(k[:-len(".comfy_quant")] for k in sd.keys() if k.endswith(".comfy_quant"))
|
| 125 |
+
quant_set = set(quant_prefixes)
|
| 126 |
+
|
| 127 |
+
# 1. Load all non-quantized parameters/buffers first
|
| 128 |
+
regular_sd = {}
|
| 129 |
+
for k, v in sd.items():
|
| 130 |
+
if k.endswith((".comfy_quant", ".weight_scale", ".weight_s_rel", ".weight_s_channel", ".weight_codebook")):
|
| 131 |
+
continue
|
| 132 |
+
if k.endswith(".weight") and k[:-7] in quant_set:
|
| 133 |
+
continue
|
| 134 |
+
if k.endswith(".bias") and k[:-5] in quant_set:
|
| 135 |
+
continue
|
| 136 |
+
regular_sd[k] = v
|
| 137 |
+
|
| 138 |
+
missing, unexpected = module.load_state_dict(regular_sd, strict=False, assign=True)
|
| 139 |
+
# 2. Replace each quantized Linear with ConvRotInt8Linear or ConvRotPackedLinear
|
| 140 |
+
counts = {"int8": 0, "w6a8_int8": 0, "asym_w4a8_int8": 0}
|
| 141 |
+
for prefix in quant_prefixes:
|
| 142 |
+
conf = _decode_comfy_quant(sd[f"{prefix}.comfy_quant"])
|
| 143 |
+
fmt = conf["format"]
|
| 144 |
+
bias = sd.get(f"{prefix}.bias", None)
|
| 145 |
+
if bias is not None:
|
| 146 |
+
bias = bias.to(device=device, dtype=dtype)
|
| 147 |
+
if fmt == "int8_tensorwise":
|
| 148 |
+
layer = ConvRotInt8Linear(
|
| 149 |
+
weight=sd[f"{prefix}.weight"].to(device=device),
|
| 150 |
+
weight_scale=sd[f"{prefix}.weight_scale"].to(device=device),
|
| 151 |
+
bias=bias,
|
| 152 |
+
convrot=conf.get("convrot", True),
|
| 153 |
+
convrot_groupsize=conf.get("convrot_groupsize", 256),
|
| 154 |
+
)
|
| 155 |
+
counts["int8"] += 1
|
| 156 |
+
elif fmt in ("w6a8_int8", "asym_w4a8_int8"):
|
| 157 |
+
cb = sd.get(f"{prefix}.weight_codebook", None)
|
| 158 |
+
layer = ConvRotPackedLinear(
|
| 159 |
+
qdata=sd[f"{prefix}.weight"].to(device=device),
|
| 160 |
+
s_rel=sd[f"{prefix}.weight_s_rel"].to(device=device),
|
| 161 |
+
s_channel=sd[f"{prefix}.weight_s_channel"].to(device=device),
|
| 162 |
+
codebook=cb.to(device=device) if cb is not None else None,
|
| 163 |
+
bias=bias,
|
| 164 |
+
group_size=conf.get("group_size", 16),
|
| 165 |
+
convrot_groupsize=conf.get("convrot_groupsize", 256),
|
| 166 |
+
)
|
| 167 |
+
counts[fmt] += 1
|
| 168 |
+
else:
|
| 169 |
+
raise ValueError(f"Unsupported comfy_quant format: {fmt}")
|
| 170 |
+
_set_submodule(module, prefix, layer)
|
| 171 |
+
|
| 172 |
+
# Materialize any non-persistent rotary_emb buffers still on meta
|
| 173 |
+
for m_name, mod in list(module.named_modules()):
|
| 174 |
+
if any(b.device.type == "meta" for b in mod.buffers(recurse=False)):
|
| 175 |
+
if hasattr(mod, "config"):
|
| 176 |
+
_set_submodule(module, m_name, mod.__class__(config=mod.config).to(device))
|
| 177 |
+
|
| 178 |
+
module.to(device=device)
|
| 179 |
+
del sd, regular_sd
|
| 180 |
+
return {"counts": counts, "missing_nonquant": [m for m in missing if m[:-7] not in quant_set and m[:-5] not in quant_set]}
|
| 181 |
+
|
| 182 |
+
|
| 183 |
+
def load_glm_image_convrot(
|
| 184 |
+
base_model: str,
|
| 185 |
+
transformer_path: Optional[str] = None,
|
| 186 |
+
vlm_path: Optional[str] = None,
|
| 187 |
+
device: str = "cuda",
|
| 188 |
+
dtype: torch.dtype = torch.bfloat16,
|
| 189 |
+
) -> tuple[GlmImagePipeline, dict]:
|
| 190 |
+
"""Load GlmImagePipeline with ConvRot quantized transformer and/or vision_language_encoder."""
|
| 191 |
+
t0 = time.perf_counter()
|
| 192 |
+
from diffusers import GlmImageTransformer2DModel
|
| 193 |
+
from transformers import GlmImageForConditionalGeneration, AutoConfig
|
| 194 |
+
|
| 195 |
+
kwargs = {"dtype": dtype}
|
| 196 |
+
info = {}
|
| 197 |
+
|
| 198 |
+
if transformer_path is not None:
|
| 199 |
+
tcfg_path = os.path.join(base_model, "transformer", "config.json")
|
| 200 |
+
tcfg = json.load(open(tcfg_path, "r", encoding="utf-8"))
|
| 201 |
+
with torch.device("meta"):
|
| 202 |
+
transformer = GlmImageTransformer2DModel.from_config(tcfg)
|
| 203 |
+
info["transformer"] = apply_convrot_state_dict(transformer, transformer_path, device=device, dtype=dtype)
|
| 204 |
+
kwargs["transformer"] = transformer
|
| 205 |
+
|
| 206 |
+
if vlm_path is not None:
|
| 207 |
+
vcfg = AutoConfig.from_pretrained(os.path.join(base_model, "vision_language_encoder"))
|
| 208 |
+
with torch.device("meta"):
|
| 209 |
+
vlm = GlmImageForConditionalGeneration(vcfg)
|
| 210 |
+
info["vision_language_encoder"] = apply_convrot_state_dict(vlm, vlm_path, device=device, dtype=dtype)
|
| 211 |
+
kwargs["vision_language_encoder"] = vlm
|
| 212 |
+
|
| 213 |
+
pipe = GlmImagePipeline.from_pretrained(base_model, **kwargs)
|
| 214 |
+
pipe.to(device)
|
| 215 |
+
torch.cuda.synchronize()
|
| 216 |
+
info["load_s"] = round(time.perf_counter() - t0, 3)
|
| 217 |
+
return pipe, info
|
int4/glm-image-transformer-int4-convrot.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e7f39877a0fcbddffa55a406df17d0282eca23d80fa17bfa66f02ac7d17db394
|
| 3 |
+
size 4427074860
|
int4/glm-image-vlm-int4-convrot.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:eb330536d7912aa973aa69323651d4d4f02aad1cb2ee6bc0fb57285f51f63eac
|
| 3 |
+
size 8480157504
|
int6/glm-image-transformer-int6-convrot.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:3334be3d3a53f7d539cc002f6c7d55a2aadb42b0b73aafcb0f7537ea518d91c1
|
| 3 |
+
size 5936991536
|
int6/glm-image-vlm-int6-convrot.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:1174be84e21128cb3796792c6d7fef886f3193e960a5ecc122b0496b0c88468d
|
| 3 |
+
size 9937654576
|
int8/glm-image-transformer-int8-convrot.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f0cbe06ab2892232baaadfc2a1432d470ecdf0952da7059de12f06cf7b65ac27
|
| 3 |
+
size 7069428600
|
int8/glm-image-vlm-int8-convrot.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ae3c3149d8a364fb7160b8395d86c6db6919defc01545d6e4edb3510a7ad8a1d
|
| 3 |
+
size 11030777376
|
metrics/g4_glm_bf16_stats.json
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"load_s": 4.51,
|
| 3 |
+
"allocated_vram_gb": 35.41,
|
| 4 |
+
"peak_vram_gb": 40.55,
|
| 5 |
+
"cases": {
|
| 6 |
+
"t2i_cabin_s42": {
|
| 7 |
+
"total_s": 48.89,
|
| 8 |
+
"ar_s": 24.39,
|
| 9 |
+
"dit_s": 24.49,
|
| 10 |
+
"peak_vram_gb": 40.55
|
| 11 |
+
},
|
| 12 |
+
"t2i_poster_s1337": {
|
| 13 |
+
"total_s": 45.62,
|
| 14 |
+
"ar_s": 23.97,
|
| 15 |
+
"dit_s": 21.65,
|
| 16 |
+
"peak_vram_gb": 40.55
|
| 17 |
+
},
|
| 18 |
+
"t2i_watch_s777": {
|
| 19 |
+
"total_s": 45.51,
|
| 20 |
+
"ar_s": 23.96,
|
| 21 |
+
"dit_s": 21.55,
|
| 22 |
+
"peak_vram_gb": 40.55
|
| 23 |
+
}
|
| 24 |
+
}
|
| 25 |
+
}
|
metrics/g4_glm_int4_stats.json
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"load_s": 19.055,
|
| 3 |
+
"allocated_vram_gb": 14.257,
|
| 4 |
+
"transformer_bytes": 4427074860,
|
| 5 |
+
"vlm_bytes": 8480157504,
|
| 6 |
+
"load_info": {
|
| 7 |
+
"transformer": {
|
| 8 |
+
"counts": {
|
| 9 |
+
"int8": 30,
|
| 10 |
+
"w6a8_int8": 0,
|
| 11 |
+
"asym_w4a8_int8": 180
|
| 12 |
+
},
|
| 13 |
+
"missing_nonquant": []
|
| 14 |
+
},
|
| 15 |
+
"vision_language_encoder": {
|
| 16 |
+
"counts": {
|
| 17 |
+
"int8": 200,
|
| 18 |
+
"w6a8_int8": 0,
|
| 19 |
+
"asym_w4a8_int8": 120
|
| 20 |
+
},
|
| 21 |
+
"missing_nonquant": []
|
| 22 |
+
},
|
| 23 |
+
"load_s": 19.055
|
| 24 |
+
},
|
| 25 |
+
"cases": {
|
| 26 |
+
"t2i_cabin_s42": {
|
| 27 |
+
"total_s": 49.516,
|
| 28 |
+
"ar_s": 31.199,
|
| 29 |
+
"dit_s": 18.317,
|
| 30 |
+
"peak_vram_gb": 19.399,
|
| 31 |
+
"ar_token_match": 0.15918,
|
| 32 |
+
"e2e": {
|
| 33 |
+
"cos": 0.75155,
|
| 34 |
+
"ssim": 0.38994,
|
| 35 |
+
"psnr": 10.135
|
| 36 |
+
},
|
| 37 |
+
"dit_only": {
|
| 38 |
+
"cos": 0.99774,
|
| 39 |
+
"ssim": 0.97232,
|
| 40 |
+
"psnr": 30.439
|
| 41 |
+
}
|
| 42 |
+
},
|
| 43 |
+
"t2i_poster_s1337": {
|
| 44 |
+
"total_s": 48.78,
|
| 45 |
+
"ar_s": 30.74,
|
| 46 |
+
"dit_s": 18.039,
|
| 47 |
+
"peak_vram_gb": 19.399,
|
| 48 |
+
"ar_token_match": 0.27148,
|
| 49 |
+
"e2e": {
|
| 50 |
+
"cos": 0.90623,
|
| 51 |
+
"ssim": 0.44728,
|
| 52 |
+
"psnr": 10.356
|
| 53 |
+
},
|
| 54 |
+
"dit_only": {
|
| 55 |
+
"cos": 0.99911,
|
| 56 |
+
"ssim": 0.96503,
|
| 57 |
+
"psnr": 30.551
|
| 58 |
+
}
|
| 59 |
+
},
|
| 60 |
+
"t2i_watch_s777": {
|
| 61 |
+
"total_s": 48.752,
|
| 62 |
+
"ar_s": 30.956,
|
| 63 |
+
"dit_s": 17.796,
|
| 64 |
+
"peak_vram_gb": 19.399,
|
| 65 |
+
"ar_token_match": 0.12695,
|
| 66 |
+
"e2e": {
|
| 67 |
+
"cos": 0.67734,
|
| 68 |
+
"ssim": 0.32457,
|
| 69 |
+
"psnr": 9.743
|
| 70 |
+
},
|
| 71 |
+
"dit_only": {
|
| 72 |
+
"cos": 0.99211,
|
| 73 |
+
"ssim": 0.96084,
|
| 74 |
+
"psnr": 26.367
|
| 75 |
+
}
|
| 76 |
+
}
|
| 77 |
+
}
|
| 78 |
+
}
|
metrics/g4_glm_int6_stats.json
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"load_s": 57.556,
|
| 3 |
+
"allocated_vram_gb": 17.164,
|
| 4 |
+
"transformer_bytes": 5936991536,
|
| 5 |
+
"vlm_bytes": 9937654576,
|
| 6 |
+
"load_info": {
|
| 7 |
+
"transformer": {
|
| 8 |
+
"counts": {
|
| 9 |
+
"int8": 30,
|
| 10 |
+
"w6a8_int8": 180,
|
| 11 |
+
"asym_w4a8_int8": 0
|
| 12 |
+
},
|
| 13 |
+
"missing_nonquant": []
|
| 14 |
+
},
|
| 15 |
+
"vision_language_encoder": {
|
| 16 |
+
"counts": {
|
| 17 |
+
"int8": 200,
|
| 18 |
+
"w6a8_int8": 120,
|
| 19 |
+
"asym_w4a8_int8": 0
|
| 20 |
+
},
|
| 21 |
+
"missing_nonquant": []
|
| 22 |
+
},
|
| 23 |
+
"load_s": 57.556
|
| 24 |
+
},
|
| 25 |
+
"cases": {
|
| 26 |
+
"t2i_cabin_s42": {
|
| 27 |
+
"total_s": 49.203,
|
| 28 |
+
"ar_s": 30.699,
|
| 29 |
+
"dit_s": 18.505,
|
| 30 |
+
"peak_vram_gb": 22.307,
|
| 31 |
+
"ar_token_match": 0.2041,
|
| 32 |
+
"e2e": {
|
| 33 |
+
"cos": 0.82284,
|
| 34 |
+
"ssim": 0.44566,
|
| 35 |
+
"psnr": 11.74
|
| 36 |
+
},
|
| 37 |
+
"dit_only": {
|
| 38 |
+
"cos": 0.99985,
|
| 39 |
+
"ssim": 0.99664,
|
| 40 |
+
"psnr": 41.739
|
| 41 |
+
}
|
| 42 |
+
},
|
| 43 |
+
"t2i_poster_s1337": {
|
| 44 |
+
"total_s": 49.033,
|
| 45 |
+
"ar_s": 30.852,
|
| 46 |
+
"dit_s": 18.181,
|
| 47 |
+
"peak_vram_gb": 22.307,
|
| 48 |
+
"ar_token_match": 0.26465,
|
| 49 |
+
"e2e": {
|
| 50 |
+
"cos": 0.87723,
|
| 51 |
+
"ssim": 0.44563,
|
| 52 |
+
"psnr": 9.231
|
| 53 |
+
},
|
| 54 |
+
"dit_only": {
|
| 55 |
+
"cos": 0.9999,
|
| 56 |
+
"ssim": 0.99405,
|
| 57 |
+
"psnr": 39.734
|
| 58 |
+
}
|
| 59 |
+
},
|
| 60 |
+
"t2i_watch_s777": {
|
| 61 |
+
"total_s": 48.709,
|
| 62 |
+
"ar_s": 30.755,
|
| 63 |
+
"dit_s": 17.954,
|
| 64 |
+
"peak_vram_gb": 22.307,
|
| 65 |
+
"ar_token_match": 0.2334,
|
| 66 |
+
"e2e": {
|
| 67 |
+
"cos": 0.69718,
|
| 68 |
+
"ssim": 0.24256,
|
| 69 |
+
"psnr": 8.985
|
| 70 |
+
},
|
| 71 |
+
"dit_only": {
|
| 72 |
+
"cos": 0.99926,
|
| 73 |
+
"ssim": 0.99332,
|
| 74 |
+
"psnr": 36.681
|
| 75 |
+
}
|
| 76 |
+
}
|
| 77 |
+
}
|
| 78 |
+
}
|
metrics/g4_glm_int8_stats.json
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"load_s": 8.905,
|
| 3 |
+
"allocated_vram_gb": 19.445,
|
| 4 |
+
"transformer_bytes": 7069428600,
|
| 5 |
+
"vlm_bytes": 11030777376,
|
| 6 |
+
"load_info": {
|
| 7 |
+
"transformer": {
|
| 8 |
+
"counts": {
|
| 9 |
+
"int8": 210,
|
| 10 |
+
"w6a8_int8": 0,
|
| 11 |
+
"asym_w4a8_int8": 0
|
| 12 |
+
},
|
| 13 |
+
"missing_nonquant": []
|
| 14 |
+
},
|
| 15 |
+
"vision_language_encoder": {
|
| 16 |
+
"counts": {
|
| 17 |
+
"int8": 320,
|
| 18 |
+
"w6a8_int8": 0,
|
| 19 |
+
"asym_w4a8_int8": 0
|
| 20 |
+
},
|
| 21 |
+
"missing_nonquant": []
|
| 22 |
+
},
|
| 23 |
+
"load_s": 8.905
|
| 24 |
+
},
|
| 25 |
+
"cases": {
|
| 26 |
+
"t2i_cabin_s42": {
|
| 27 |
+
"total_s": 47.251,
|
| 28 |
+
"ar_s": 29.312,
|
| 29 |
+
"dit_s": 17.939,
|
| 30 |
+
"peak_vram_gb": 24.588,
|
| 31 |
+
"ar_token_match": 0.25293,
|
| 32 |
+
"e2e": {
|
| 33 |
+
"cos": 0.85584,
|
| 34 |
+
"ssim": 0.50515,
|
| 35 |
+
"psnr": 12.582
|
| 36 |
+
},
|
| 37 |
+
"dit_only": {
|
| 38 |
+
"cos": 0.99996,
|
| 39 |
+
"ssim": 0.99821,
|
| 40 |
+
"psnr": 47.207
|
| 41 |
+
}
|
| 42 |
+
},
|
| 43 |
+
"t2i_poster_s1337": {
|
| 44 |
+
"total_s": 46.541,
|
| 45 |
+
"ar_s": 29.225,
|
| 46 |
+
"dit_s": 17.316,
|
| 47 |
+
"peak_vram_gb": 24.588,
|
| 48 |
+
"ar_token_match": 0.19727,
|
| 49 |
+
"e2e": {
|
| 50 |
+
"cos": 0.85398,
|
| 51 |
+
"ssim": 0.41511,
|
| 52 |
+
"psnr": 8.628
|
| 53 |
+
},
|
| 54 |
+
"dit_only": {
|
| 55 |
+
"cos": 0.99996,
|
| 56 |
+
"ssim": 0.99647,
|
| 57 |
+
"psnr": 42.833
|
| 58 |
+
}
|
| 59 |
+
},
|
| 60 |
+
"t2i_watch_s777": {
|
| 61 |
+
"total_s": 46.298,
|
| 62 |
+
"ar_s": 29.085,
|
| 63 |
+
"dit_s": 17.213,
|
| 64 |
+
"peak_vram_gb": 24.588,
|
| 65 |
+
"ar_token_match": 0.20215,
|
| 66 |
+
"e2e": {
|
| 67 |
+
"cos": 0.69522,
|
| 68 |
+
"ssim": 0.20796,
|
| 69 |
+
"psnr": 8.432
|
| 70 |
+
},
|
| 71 |
+
"dit_only": {
|
| 72 |
+
"cos": 0.99953,
|
| 73 |
+
"ssim": 0.99567,
|
| 74 |
+
"psnr": 38.65
|
| 75 |
+
}
|
| 76 |
+
}
|
| 77 |
+
}
|
| 78 |
+
}
|
metrics/g4_glm_vlm_teacher_forced.json
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"int8": {
|
| 3 |
+
"mean_kl": 0.08076,
|
| 4 |
+
"mean_top1": 0.8971,
|
| 5 |
+
"per_top1": [
|
| 6 |
+
0.93555,
|
| 7 |
+
0.82031,
|
| 8 |
+
0.93555
|
| 9 |
+
]
|
| 10 |
+
},
|
| 11 |
+
"int6": {
|
| 12 |
+
"mean_kl": 0.08223,
|
| 13 |
+
"mean_top1": 0.8919,
|
| 14 |
+
"per_top1": [
|
| 15 |
+
0.90039,
|
| 16 |
+
0.84961,
|
| 17 |
+
0.92578
|
| 18 |
+
]
|
| 19 |
+
},
|
| 20 |
+
"int4": {
|
| 21 |
+
"mean_kl": 0.31624,
|
| 22 |
+
"mean_top1": 0.7962,
|
| 23 |
+
"per_top1": [
|
| 24 |
+
0.8418,
|
| 25 |
+
0.70898,
|
| 26 |
+
0.83789
|
| 27 |
+
]
|
| 28 |
+
}
|
| 29 |
+
}
|
scripts/eval_glm_image.py
ADDED
|
@@ -0,0 +1,161 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Evaluate GLM-Image ConvRot INT8 / INT6 / INT4 checkpoints against BF16 baseline (end-to-end + DiT-isolated)."""
|
| 2 |
+
import argparse
|
| 3 |
+
import gc
|
| 4 |
+
import json
|
| 5 |
+
import os
|
| 6 |
+
import time
|
| 7 |
+
|
| 8 |
+
import numpy as np
|
| 9 |
+
import torch
|
| 10 |
+
from PIL import Image
|
| 11 |
+
from skimage.metrics import peak_signal_noise_ratio, structural_similarity
|
| 12 |
+
|
| 13 |
+
from glm_image_convrot import load_glm_image_convrot
|
| 14 |
+
|
| 15 |
+
BASE_MODEL = "/content/models/GLM-Image"
|
| 16 |
+
BF16_DIR = "/content/work/glm_bf16"
|
| 17 |
+
|
| 18 |
+
EVAL_CASES = [
|
| 19 |
+
(
|
| 20 |
+
"t2i_cabin_s42",
|
| 21 |
+
"A cozy wooden cabin beside a calm alpine lake at golden hour, warm glowing windows, pine trees reflected in crystal clear water, photorealistic, sharp detail",
|
| 22 |
+
1024,
|
| 23 |
+
1024,
|
| 24 |
+
42,
|
| 25 |
+
),
|
| 26 |
+
(
|
| 27 |
+
"t2i_poster_s1337",
|
| 28 |
+
"A vintage travel poster for 'ALPS EXPRESS' with bold cream typography at the top, a red steam train crossing a stone viaduct over snowy mountains, and small text 'SWITZERLAND 1964' at the bottom, crisp retro print illustration",
|
| 29 |
+
1024,
|
| 30 |
+
1024,
|
| 31 |
+
1337,
|
| 32 |
+
),
|
| 33 |
+
(
|
| 34 |
+
"t2i_watch_s777",
|
| 35 |
+
"Macro studio photograph of a luxury mechanical wristwatch on dark slate stone, brushed steel case, sapphire crystal catching soft rim light, crisp numerals and gears",
|
| 36 |
+
1024,
|
| 37 |
+
1024,
|
| 38 |
+
777,
|
| 39 |
+
),
|
| 40 |
+
]
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def img_metrics(ref_img: Image.Image, cand_img: Image.Image) -> dict:
|
| 44 |
+
a = np.asarray(ref_img.convert("RGB"), dtype=np.float32) / 255.0
|
| 45 |
+
b = np.asarray(cand_img.convert("RGB"), dtype=np.float32) / 255.0
|
| 46 |
+
cos = float(np.dot(a.ravel(), b.ravel()) / (np.linalg.norm(a.ravel()) * np.linalg.norm(b.ravel()) + 1e-12))
|
| 47 |
+
ssim = float(structural_similarity(a, b, channel_axis=2, data_range=1.0))
|
| 48 |
+
psnr = float(peak_signal_noise_ratio(a, b, data_range=1.0))
|
| 49 |
+
return {"cos": round(cos, 5), "ssim": round(ssim, 5), "psnr": round(psnr, 3)}
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def main():
|
| 53 |
+
ap = argparse.ArgumentParser()
|
| 54 |
+
ap.add_argument("--transformer", default=None)
|
| 55 |
+
ap.add_argument("--vlm", default=None)
|
| 56 |
+
ap.add_argument("--out", required=True)
|
| 57 |
+
ap.add_argument("--skip-dit-isolated", action="store_true")
|
| 58 |
+
args = ap.parse_args()
|
| 59 |
+
|
| 60 |
+
os.makedirs(args.out, exist_ok=True)
|
| 61 |
+
torch.cuda.reset_peak_memory_stats()
|
| 62 |
+
pipe, load_info = load_glm_image_convrot(
|
| 63 |
+
BASE_MODEL,
|
| 64 |
+
transformer_path=args.transformer,
|
| 65 |
+
vlm_path=args.vlm,
|
| 66 |
+
device="cuda",
|
| 67 |
+
dtype=torch.bfloat16,
|
| 68 |
+
)
|
| 69 |
+
alloc_gb = torch.cuda.memory_allocated() / 1e9
|
| 70 |
+
print(f"[EVAL] Loaded in {load_info['load_s']:.2f}s | allocated={alloc_gb:.2f} GB | {load_info}", flush=True)
|
| 71 |
+
|
| 72 |
+
orig_gen = pipe.generate_prior_tokens
|
| 73 |
+
timings = {}
|
| 74 |
+
forced_prior = [None]
|
| 75 |
+
|
| 76 |
+
def hooked_gen(*g_args, **g_kwargs):
|
| 77 |
+
if forced_prior[0] is not None:
|
| 78 |
+
return forced_prior[0]
|
| 79 |
+
torch.cuda.synchronize()
|
| 80 |
+
ts = time.perf_counter()
|
| 81 |
+
out = orig_gen(*g_args, **g_kwargs)
|
| 82 |
+
torch.cuda.synchronize()
|
| 83 |
+
timings["ar_s"] = time.perf_counter() - ts
|
| 84 |
+
timings["prior_out"] = [t.detach().cpu() if torch.is_tensor(t) else t for t in out] if isinstance(out, (list, tuple)) else out.detach().cpu()
|
| 85 |
+
return out
|
| 86 |
+
|
| 87 |
+
pipe.generate_prior_tokens = hooked_gen
|
| 88 |
+
|
| 89 |
+
stats = {
|
| 90 |
+
"load_s": load_info["load_s"],
|
| 91 |
+
"allocated_vram_gb": round(alloc_gb, 3),
|
| 92 |
+
"transformer_bytes": os.path.getsize(args.transformer) if args.transformer else None,
|
| 93 |
+
"vlm_bytes": os.path.getsize(args.vlm) if args.vlm else None,
|
| 94 |
+
"load_info": load_info,
|
| 95 |
+
"cases": {},
|
| 96 |
+
}
|
| 97 |
+
|
| 98 |
+
for tag, prompt, h, w, seed in EVAL_CASES:
|
| 99 |
+
ref_img = Image.open(os.path.join(BF16_DIR, f"{tag}.png"))
|
| 100 |
+
ref_prior = torch.load(os.path.join(BF16_DIR, f"{tag}_prior.pt"), map_location="cpu", weights_only=False)
|
| 101 |
+
|
| 102 |
+
# 1. End-to-end (Quant AR + Quant DiT)
|
| 103 |
+
forced_prior[0] = None
|
| 104 |
+
timings.clear()
|
| 105 |
+
torch.cuda.reset_peak_memory_stats()
|
| 106 |
+
gen = torch.Generator(device="cuda").manual_seed(seed)
|
| 107 |
+
t0 = time.perf_counter()
|
| 108 |
+
res = pipe(prompt=prompt, height=h, width=w, num_inference_steps=50, guidance_scale=1.5, generator=gen)
|
| 109 |
+
torch.cuda.synchronize()
|
| 110 |
+
total_s = time.perf_counter() - t0
|
| 111 |
+
peak_gb = torch.cuda.max_memory_allocated() / 1e9
|
| 112 |
+
img = res.images[0]
|
| 113 |
+
img.save(os.path.join(args.out, f"{tag}.png"))
|
| 114 |
+
m_e2e = img_metrics(ref_img, img)
|
| 115 |
+
|
| 116 |
+
# Compare AR prior tokens vs BF16 prior tokens
|
| 117 |
+
cand_prior = timings.get("prior_out")
|
| 118 |
+
ar_exact = None
|
| 119 |
+
if isinstance(ref_prior, (list, tuple)) and isinstance(cand_prior, (list, tuple)) and torch.is_tensor(ref_prior[0]) and torch.is_tensor(cand_prior[0]):
|
| 120 |
+
rp, cp_t = ref_prior[0].view(-1), cand_prior[0].view(-1)
|
| 121 |
+
n_min = min(rp.numel(), cp_t.numel())
|
| 122 |
+
ar_exact = round(float((rp[:n_min] == cp_t[:n_min]).float().mean()), 5)
|
| 123 |
+
|
| 124 |
+
ar_s = timings.get("ar_s", 0.0)
|
| 125 |
+
dit_s = max(0.0, total_s - ar_s)
|
| 126 |
+
|
| 127 |
+
case_stat = {
|
| 128 |
+
"total_s": round(total_s, 3),
|
| 129 |
+
"ar_s": round(ar_s, 3),
|
| 130 |
+
"dit_s": round(dit_s, 3),
|
| 131 |
+
"peak_vram_gb": round(peak_gb, 3),
|
| 132 |
+
"ar_token_match": ar_exact,
|
| 133 |
+
"e2e": m_e2e,
|
| 134 |
+
}
|
| 135 |
+
|
| 136 |
+
# 2. DiT-isolated (exact BF16 prior tokens -> Quant DiT)
|
| 137 |
+
if not args.skip_dit_isolated and isinstance(ref_prior, (list, tuple)):
|
| 138 |
+
forced_prior[0] = tuple(t.to("cuda") if torch.is_tensor(t) else t for t in ref_prior)
|
| 139 |
+
gen = torch.Generator(device="cuda").manual_seed(seed)
|
| 140 |
+
res_dit = pipe(prompt=prompt, height=h, width=w, num_inference_steps=50, guidance_scale=1.5, generator=gen)
|
| 141 |
+
torch.cuda.synchronize()
|
| 142 |
+
dit_img = res_dit.images[0]
|
| 143 |
+
dit_img.save(os.path.join(args.out, f"{tag}_dit_only.png"))
|
| 144 |
+
case_stat["dit_only"] = img_metrics(ref_img, dit_img)
|
| 145 |
+
forced_prior[0] = None
|
| 146 |
+
|
| 147 |
+
stats["cases"][tag] = case_stat
|
| 148 |
+
print(
|
| 149 |
+
f" [{tag}] total={total_s:.2f}s (AR={ar_s:.2f}s, DiT={dit_s:.2f}s) vram={peak_gb:.2f}GB | "
|
| 150 |
+
f"AR_match={ar_exact} | E2E: cos={m_e2e['cos']} ssim={m_e2e['ssim']} psnr={m_e2e['psnr']}dB | "
|
| 151 |
+
f"DiT_only: {case_stat.get('dit_only')}",
|
| 152 |
+
flush=True,
|
| 153 |
+
)
|
| 154 |
+
|
| 155 |
+
with open(os.path.join(args.out, "stats.json"), "w") as f:
|
| 156 |
+
json.dump(stats, f, indent=2)
|
| 157 |
+
print("EVAL_GLM_DONE", flush=True)
|
| 158 |
+
|
| 159 |
+
|
| 160 |
+
if __name__ == "__main__":
|
| 161 |
+
main()
|