Spaces:
Running on Zero
Running on Zero
P-H3-Image demo
Browse files- README.md +10 -8
- adaln_calls.json +1 -0
- app.py +246 -0
- examples.py +84 -0
- p_h3_image_fast.py +857 -0
- requirements.txt +11 -0
README.md
CHANGED
|
@@ -1,13 +1,15 @@
|
|
| 1 |
---
|
| 2 |
-
title: P
|
| 3 |
-
emoji:
|
| 4 |
-
colorFrom:
|
| 5 |
-
colorTo:
|
| 6 |
sdk: gradio
|
| 7 |
-
sdk_version: 6.
|
| 8 |
-
python_version:
|
| 9 |
app_file: app.py
|
| 10 |
pinned: false
|
|
|
|
|
|
|
|
|
|
|
|
|
| 11 |
---
|
| 12 |
-
|
| 13 |
-
Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
|
|
|
|
| 1 |
---
|
| 2 |
+
title: P-H3-Image
|
| 3 |
+
emoji: 🖼️
|
| 4 |
+
colorFrom: gray
|
| 5 |
+
colorTo: gray
|
| 6 |
sdk: gradio
|
| 7 |
+
sdk_version: 6.28.0
|
| 8 |
+
python_version: "3.12"
|
| 9 |
app_file: app.py
|
| 10 |
pinned: false
|
| 11 |
+
startup_duration_timeout: 1h
|
| 12 |
+
short_description: Text-to-image and image editing, based on MiniMax-H3
|
| 13 |
+
models:
|
| 14 |
+
- PrunaAI/P-H3-Image
|
| 15 |
---
|
|
|
|
|
|
adaln_calls.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
[[0.0], [0.0, 0.9990000128746033], [0.015246987342834473], [0.015246987342834473, 0.9990000128746033], [0.03209501504898071], [0.03209501504898071, 0.9990000128746033], [0.04831200838088989], [0.04831200838088989, 0.9990000128746033], [0.07462799549102783], [0.07462799549102783, 0.9990000128746033], [0.11807698011398315], [0.11807698011398315, 0.9990000128746033], [0.18991702795028687], [0.18991702795028687, 0.9990000128746033], [0.4712820053100586], [0.4712820053100586, 0.9990000128746033]]
|
app.py
ADDED
|
@@ -0,0 +1,246 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""P-H3-Image on ZeroGPU (`xlarge`: a full RTX PRO 6000 Blackwell, 96 GB).
|
| 2 |
+
|
| 3 |
+
The pipeline is loaded once at startup, one component at a time (the host has ~97 GB), and slimmed to fit the slice
|
| 4 |
+
with p_h3_image_fast.py (kernels only, no torch.compile): AdaLN tables instead of the transformer's AdaLN projections,
|
| 5 |
+
FP8 linears everywhere, fused Triton DiT blocks, batched VAE tiles, and for the prompt upsampler a weight-only FP8 GEMV,
|
| 6 |
+
decode attention and speculative decoding with its MTP head in one CUDA graph per step.
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
import json
|
| 10 |
+
import os
|
| 11 |
+
import random
|
| 12 |
+
import tempfile
|
| 13 |
+
import time
|
| 14 |
+
|
| 15 |
+
os.environ.setdefault("DIFFUSERS_TRUST_REMOTE_KERNELS", "true")
|
| 16 |
+
os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
|
| 17 |
+
|
| 18 |
+
import gradio as gr
|
| 19 |
+
import spaces
|
| 20 |
+
import torch
|
| 21 |
+
from huggingface_hub import snapshot_download
|
| 22 |
+
from PIL import Image, ImageOps
|
| 23 |
+
from torchvision.io import encode_jpeg
|
| 24 |
+
|
| 25 |
+
import p_h3_image_fast as fast
|
| 26 |
+
from diffusers import ModularPipeline
|
| 27 |
+
from examples import EXAMPLES
|
| 28 |
+
|
| 29 |
+
REPO, REVISION = "PrunaAI/P-H3-Image", "ee4a685de1fbd4337a50bcf9b3c407e4f8463d49"
|
| 30 |
+
# Canvases (width, height), multiples of 32: 4 and 8 MP as tested, 16 MP from the same ratios.
|
| 31 |
+
RATIOS = {"1:1": 1.0, "4:3": 4 / 3, "3:4": 3 / 4, "3:2": 3 / 2, "2:3": 2 / 3, "16:9": 16 / 9, "9:16": 9 / 16}
|
| 32 |
+
T2I_RESOLUTIONS, EDIT_RESOLUTIONS = ["4 MP", "8 MP", "16 MP"], ["4 MP", "8 MP"]
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def _canvas(ratio: float, megapixels: int) -> tuple[int, int]:
|
| 36 |
+
pixels = megapixels * 1024**2
|
| 37 |
+
return round((pixels * ratio) ** 0.5 / 32) * 32, round((pixels / ratio) ** 0.5 / 32) * 32
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
SIZES = {
|
| 41 |
+
"4 MP": {"1:1": (2048, 2048), "4:3": (2368, 1760), "3:4": (1760, 2368), "3:2": (2496, 1664), "2:3": (1664, 2496),
|
| 42 |
+
"16:9": (2720, 1536), "9:16": (1536, 2720)},
|
| 43 |
+
"8 MP": {"1:1": (2912, 2912), "4:3": (3360, 2496), "3:4": (2496, 3360), "3:2": (3552, 2368), "2:3": (2368, 3552),
|
| 44 |
+
"16:9": (3872, 2176), "9:16": (2176, 3872)},
|
| 45 |
+
"16 MP": {name: _canvas(ratio, 16) for name, ratio in RATIOS.items()},
|
| 46 |
+
}
|
| 47 |
+
MAX_IMAGES = 8
|
| 48 |
+
# All references together stay within 8 MP: one image keeps up to 8 MP, eight get 1 MP each.
|
| 49 |
+
REFERENCE_BUDGET = 8 * 1024**2
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def load_pipeline():
|
| 53 |
+
path = snapshot_download(REPO, revision=REVISION, max_workers=16)
|
| 54 |
+
with open(os.path.join(os.path.dirname(__file__), "adaln_calls.json")) as handle:
|
| 55 |
+
calls = json.load(handle)
|
| 56 |
+
pipe, mtp = fast.load_slim_pipeline(path, calls, log=lambda message: print(message, flush=True))
|
| 57 |
+
# The same components without the prompt upsampler step, for requests that skip upsampling.
|
| 58 |
+
blocks = ModularPipeline.from_pretrained(path).blocks
|
| 59 |
+
blocks.sub_blocks.pop("prompt_upsampler")
|
| 60 |
+
plain = blocks.init_pipeline(path)
|
| 61 |
+
plain.update_components(**{name: getattr(pipe, name) for name in plain.component_names
|
| 62 |
+
if getattr(pipe, name, None) is not None})
|
| 63 |
+
return pipe, plain, mtp
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
PIPE, PLAIN, MTP = load_pipeline()
|
| 67 |
+
UPSAMPLER_GENERATE = PIPE.prompt_upsampler.generate
|
| 68 |
+
STATE = {"kernels": False, "decoder": None, "owner": None}
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def _prepare_gpu():
|
| 72 |
+
"""Per GPU call. Once per worker process: the upsampler's hub kernels, and the speculative decoder with its CUDA
|
| 73 |
+
graph (captured on the first request, then replayed; rebuilt if the process or device changes)."""
|
| 74 |
+
upsampler = PIPE.prompt_upsampler
|
| 75 |
+
if not STATE["kernels"]:
|
| 76 |
+
from kernels import Mode
|
| 77 |
+
|
| 78 |
+
upsampler.set_use_kernels(True, mode=Mode.INFERENCE)
|
| 79 |
+
STATE["kernels"] = True
|
| 80 |
+
# The graph bakes in weight addresses: rebuild if ZeroGPU placed the weights elsewhere since the capture.
|
| 81 |
+
weights = (upsampler.lm_head.weight, MTP.fc.weight, upsampler.model.language_model.embed_tokens.weight,
|
| 82 |
+
upsampler.model.language_model.layers[0].mlp.down_proj.weight)
|
| 83 |
+
owner = (os.getpid(), torch.cuda.current_device(), tuple(w.data_ptr() for w in weights))
|
| 84 |
+
if STATE["decoder"] is None or STATE["owner"] != owner:
|
| 85 |
+
upsampler.generate = UPSAMPLER_GENERATE
|
| 86 |
+
STATE["decoder"], STATE["owner"] = fast.MTPGreedyGenerate(upsampler, MTP), owner
|
| 87 |
+
decoder = STATE["decoder"]
|
| 88 |
+
timing = {}
|
| 89 |
+
|
| 90 |
+
def generate(*args, **kwargs):
|
| 91 |
+
torch.cuda.synchronize()
|
| 92 |
+
begin = time.perf_counter()
|
| 93 |
+
out = decoder.generate(*args, **kwargs)
|
| 94 |
+
torch.cuda.synchronize()
|
| 95 |
+
timing["upsampler"] = time.perf_counter() - begin
|
| 96 |
+
timing["tokens"] = decoder.stats.get("tokens", 0)
|
| 97 |
+
return out
|
| 98 |
+
|
| 99 |
+
upsampler.generate = generate
|
| 100 |
+
return timing
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
def _duration(mode, prompt, images, aspect, resolution, upsample, seed):
|
| 104 |
+
# Measured on a full card: ~6 / 12 / 25 s for 4 / 8 / 16 MP text-to-image, edits a bit more per image. Shorter
|
| 105 |
+
# requests queue faster on ZeroGPU.
|
| 106 |
+
base = {"4 MP": 25, "8 MP": 35, "16 MP": 60}[resolution]
|
| 107 |
+
return base + (10 + 5 * len(images) if mode == "edit" else 0)
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
@spaces.GPU(size="xlarge", duration=_duration)
|
| 111 |
+
def _render(mode, prompt, images, aspect, resolution, upsample, seed):
|
| 112 |
+
timing = _prepare_gpu()
|
| 113 |
+
torch.cuda.synchronize()
|
| 114 |
+
start = time.perf_counter()
|
| 115 |
+
kwargs = {"prompt": prompt, "generator": torch.Generator().manual_seed(seed)}
|
| 116 |
+
if mode == "edit":
|
| 117 |
+
# The first image's aspect ratio at the chosen size.
|
| 118 |
+
width, height = images[0].size
|
| 119 |
+
kwargs["image"] = images
|
| 120 |
+
kwargs["width"], kwargs["height"] = _canvas(width / height, int(resolution.split()[0]))
|
| 121 |
+
else:
|
| 122 |
+
kwargs["width"], kwargs["height"] = SIZES[resolution][aspect]
|
| 123 |
+
result = (PIPE if upsample else PLAIN)(**kwargs, output_type="pt", output=["images", "prompt"])
|
| 124 |
+
path, size = _to_jpeg(result["images"])
|
| 125 |
+
total = time.perf_counter() - start
|
| 126 |
+
return path, size, result["prompt"], {"total": total, **timing}
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
def _to_jpeg(images: torch.Tensor) -> tuple[str, tuple[int, int]]:
|
| 130 |
+
"""The decoded image as a JPEG file: uint8 and nvjpeg on the GPU, so only the encoded bytes leave the worker."""
|
| 131 |
+
pixels = (images[0].clamp(0, 1) * 255).round().to(torch.uint8).contiguous()
|
| 132 |
+
try:
|
| 133 |
+
data = encode_jpeg(pixels, quality=93)
|
| 134 |
+
except Exception: # noqa: BLE001 - no nvjpeg: encode on the CPU
|
| 135 |
+
data = encode_jpeg(pixels.cpu(), quality=93)
|
| 136 |
+
with tempfile.NamedTemporaryFile(suffix=".jpg", delete=False) as handle:
|
| 137 |
+
handle.write(data.cpu().numpy().tobytes())
|
| 138 |
+
return handle.name, (pixels.shape[2], pixels.shape[1])
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
@spaces.GPU(size="xlarge", duration=300)
|
| 142 |
+
def _warmup():
|
| 143 |
+
"""Triton compiles and autotunes, and the hub kernels load, before the first visitor."""
|
| 144 |
+
image = Image.new("RGB", (1024, 1024), (120, 140, 160))
|
| 145 |
+
timing = _prepare_gpu()
|
| 146 |
+
PIPE(prompt="a lighthouse on a cliff at dusk", width=1024, height=1024, generator=torch.Generator().manual_seed(0),
|
| 147 |
+
output="images")
|
| 148 |
+
PIPE(prompt="make it night", image=[image], width=1024, height=1024, generator=torch.Generator().manual_seed(0),
|
| 149 |
+
output="images")
|
| 150 |
+
return timing
|
| 151 |
+
|
| 152 |
+
|
| 153 |
+
try:
|
| 154 |
+
_warmup()
|
| 155 |
+
except Exception as error: # noqa: BLE001 - the demo still works, the first request just pays the warmup
|
| 156 |
+
print(f"warmup failed: {error!r}", flush=True)
|
| 157 |
+
|
| 158 |
+
|
| 159 |
+
def _info(timing, size, seed):
|
| 160 |
+
upsample, tokens = timing.get("upsampler", 0.0), timing.get("tokens", 0)
|
| 161 |
+
rate = f", {tokens} tokens at {tokens / upsample:.0f} tokens/s" if upsample and tokens else ""
|
| 162 |
+
head = f"{size[0]}×{size[1]}, seed {seed} · {timing['total']:.1f} s on the GPU"
|
| 163 |
+
if not upsample:
|
| 164 |
+
return f"{head} (no prompt upsampling)"
|
| 165 |
+
return f"{head} ({upsample:.1f} s prompt upsampling{rate}, {timing['total'] - upsample:.1f} s rendering)"
|
| 166 |
+
|
| 167 |
+
|
| 168 |
+
def text_to_image(prompt, aspect, resolution, upsample):
|
| 169 |
+
if not prompt or not prompt.strip():
|
| 170 |
+
raise gr.Error("Write a prompt first.")
|
| 171 |
+
seed = random.randint(0, 2**31 - 1)
|
| 172 |
+
try:
|
| 173 |
+
path, size, upsampled, timing = _render("t2i", prompt.strip(), [], aspect, resolution, upsample, seed)
|
| 174 |
+
except RuntimeError as error:
|
| 175 |
+
raise gr.Error(str(error).split("\n")[0])
|
| 176 |
+
return path, upsampled, _info(timing, size, seed)
|
| 177 |
+
|
| 178 |
+
|
| 179 |
+
def _fit(image: Image.Image, pixels: float) -> Image.Image:
|
| 180 |
+
"""Downscale to at most `pixels` (the pipeline keeps references at their own size, from 1 MP up)."""
|
| 181 |
+
scale = (pixels / (image.width * image.height)) ** 0.5
|
| 182 |
+
if scale >= 1:
|
| 183 |
+
return image
|
| 184 |
+
return image.resize((max(1, round(image.width * scale)), max(1, round(image.height * scale))), Image.LANCZOS)
|
| 185 |
+
|
| 186 |
+
|
| 187 |
+
def edit(gallery, prompt, resolution, upsample):
|
| 188 |
+
images = []
|
| 189 |
+
for item in gallery or []:
|
| 190 |
+
item = item[0] if isinstance(item, (tuple, list)) else item
|
| 191 |
+
item = Image.open(item) if isinstance(item, str) else item
|
| 192 |
+
images.append(ImageOps.exif_transpose(item).convert("RGB"))
|
| 193 |
+
if not images:
|
| 194 |
+
raise gr.Error("Add at least one image; the first one is edited.")
|
| 195 |
+
if len(images) > MAX_IMAGES:
|
| 196 |
+
raise gr.Error(f"At most {MAX_IMAGES} images.")
|
| 197 |
+
if not prompt or not prompt.strip():
|
| 198 |
+
raise gr.Error("Describe the edit first.")
|
| 199 |
+
images = [_fit(image, REFERENCE_BUDGET / len(images)) for image in images]
|
| 200 |
+
seed = random.randint(0, 2**31 - 1)
|
| 201 |
+
try:
|
| 202 |
+
path, size, upsampled, timing = _render("edit", prompt.strip(), images, None, resolution, upsample, seed)
|
| 203 |
+
except (RuntimeError, ValueError) as error:
|
| 204 |
+
raise gr.Error(str(error).split("\n")[0])
|
| 205 |
+
return path, upsampled, _info(timing, size, seed)
|
| 206 |
+
|
| 207 |
+
|
| 208 |
+
with gr.Blocks(title="P-H3-Image") as demo:
|
| 209 |
+
gr.Markdown("# P-H3-Image\nPruna's image model for text-to-image and image editing, based on MiniMax-H3. "
|
| 210 |
+
"By default every request goes through the prompt upsampler first. "
|
| 211 |
+
"[Model card](https://huggingface.co/PrunaAI/P-H3-Image)")
|
| 212 |
+
with gr.Tab("Text to image"):
|
| 213 |
+
with gr.Row():
|
| 214 |
+
with gr.Column():
|
| 215 |
+
t_prompt = gr.Textbox(label="Prompt", lines=4)
|
| 216 |
+
with gr.Row():
|
| 217 |
+
t_aspect = gr.Dropdown(list(RATIOS), value="4:3", label="Aspect ratio")
|
| 218 |
+
t_resolution = gr.Radio(T2I_RESOLUTIONS, value="8 MP", label="Resolution")
|
| 219 |
+
t_upsample = gr.Checkbox(value=True, label="Prompt upsampling")
|
| 220 |
+
t_button = gr.Button("Generate", variant="primary")
|
| 221 |
+
with gr.Column():
|
| 222 |
+
t_image = gr.Image(label="Result", type="filepath")
|
| 223 |
+
t_info = gr.Markdown()
|
| 224 |
+
with gr.Accordion("Upsampled prompt", open=False):
|
| 225 |
+
t_upsampled = gr.Textbox(show_label=False, lines=10)
|
| 226 |
+
gr.Examples([[prompt, aspect] for _, aspect, prompt in EXAMPLES], inputs=[t_prompt, t_aspect],
|
| 227 |
+
example_labels=[label for label, _, _ in EXAMPLES])
|
| 228 |
+
t_button.click(text_to_image, [t_prompt, t_aspect, t_resolution, t_upsample], [t_image, t_upsampled, t_info])
|
| 229 |
+
with gr.Tab("Edit"):
|
| 230 |
+
with gr.Row():
|
| 231 |
+
with gr.Column():
|
| 232 |
+
e_images = gr.Gallery(label=f"Images (the first one is edited, up to {MAX_IMAGES})", type="pil",
|
| 233 |
+
columns=4, height=260, interactive=True)
|
| 234 |
+
e_prompt = gr.Textbox(label="Edit", lines=3, placeholder="e.g. put the fox from <Picture 2> on the bench")
|
| 235 |
+
e_resolution = gr.Radio(EDIT_RESOLUTIONS, value="8 MP", label="Resolution",
|
| 236 |
+
info="The first image's aspect ratio at this size.")
|
| 237 |
+
e_upsample = gr.Checkbox(value=True, label="Prompt upsampling")
|
| 238 |
+
e_button = gr.Button("Edit", variant="primary")
|
| 239 |
+
with gr.Column():
|
| 240 |
+
e_image = gr.Image(label="Result", type="filepath")
|
| 241 |
+
e_info = gr.Markdown()
|
| 242 |
+
with gr.Accordion("Upsampled prompt", open=False):
|
| 243 |
+
e_upsampled = gr.Textbox(show_label=False, lines=10)
|
| 244 |
+
e_button.click(edit, [e_images, e_prompt, e_resolution, e_upsample], [e_image, e_upsampled, e_info])
|
| 245 |
+
|
| 246 |
+
demo.queue(max_size=20).launch()
|
examples.py
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Text-to-image examples: (label, aspect ratio, prompt)."""
|
| 2 |
+
|
| 3 |
+
EXAMPLES = [
|
| 4 |
+
("Negotiation in orbit", "16:9",
|
| 5 |
+
"Medium wide shot from a low angle across a long table, on a 35mm lens, with the window and far wall behind the "
|
| 6 |
+
"figures. Five people sit in tense negotiation, leaning toward each other. Two humans in worn grey wool suits "
|
| 7 |
+
"face three alien delegates. The first alien is tall and thin-limbed, with grey skin, long folded fingers and a "
|
| 8 |
+
"robe of coarse woven fiber. The second is squat and heavy, its back plated with pale ridged shell, gripping a "
|
| 9 |
+
"stylus in three thick fingers. The third has a dark mantle of fine filaments and wears a harness of dented brass "
|
| 10 |
+
"clips. A cracked ceramic jug, scuffed data slates and curled paper printouts sit on the table. Earth fills the "
|
| 11 |
+
"large curved window, a band of blue cloud over dark ocean. Ribbed steel girders frame the foreground. Cold "
|
| 12 |
+
"sunlight rakes across the table while one amber lamp hangs overhead, casting soft shadows under the chins. Fine "
|
| 13 |
+
"dust drifts in the light. Palette of slate, bone, rust and deep teal, natural film color, fine grain, shadows "
|
| 14 |
+
"deep with detail kept."),
|
| 15 |
+
("Astronaut in a flooded jungle", "16:9",
|
| 16 |
+
"A frame from a feature film: a wide establishing shot from a low angle, with a small astronaut standing "
|
| 17 |
+
"waist-deep in a flooded jungle clearing, captured on a large-format digital cinema camera with a 35mm lens.\n\n"
|
| 18 |
+
"The astronaut wears a worn white spacesuit with grey mud on the knees, scuffed gloves, and a cracked visor that "
|
| 19 |
+
"reflects the canopy. One hand grips a rusted machete while the other rests on a broken comms pack. The helmet "
|
| 20 |
+
"is tilted toward a sound in the trees, caught mid-step as if listening for danger.\n\n"
|
| 21 |
+
"Behind the figure, tangled roots and fallen trunks rise out of black water. A pale strangler fig wraps around a "
|
| 22 |
+
"torn fragment of a landing hull. Farther back, ferns and mist-shrouded trunks fade into layers of blue-grey "
|
| 23 |
+
"haze.\n\n"
|
| 24 |
+
"The light is a low, overcast sky above the canopy, a cold top light with a faint blue-white bounce off the "
|
| 25 |
+
"water. A single battery lantern clipped to the pack casts a small warm pool across the suit.\n\n"
|
| 26 |
+
"Drifting mist hangs between the trunks, and fine rain dimples the water surface. The palette is restrained: "
|
| 27 |
+
"slate blue, steel grey, muted sage green and pale ivory, with film-like color and fine grain."),
|
| 28 |
+
("Knight in a frozen forest", "3:2",
|
| 29 |
+
"A photograph by an experienced landscape photographer, shot at eye level from a low position with a 35mm lens "
|
| 30 |
+
"at f/2.8, the knight placed in the middle distance on a narrow forest path.\n\n"
|
| 31 |
+
"The knight is a weathered man in his forties, his beard rimmed with frost and his brow tight as he scans the "
|
| 32 |
+
"trees. His dented steel plate is dulled with grey scratches, a wool cloak hangs heavy and damp at his "
|
| 33 |
+
"shoulders, and his right hand rests loosely on a sword hilt wrapped in worn leather. His breath drifts visibly "
|
| 34 |
+
"in the cold air as he walks mid-stride, one boot sinking into the snow.\n\n"
|
| 35 |
+
"Behind him, birch and pine trunks are coated in white frost, and fallen branches lie half-buried under drifts. "
|
| 36 |
+
"Moonlight comes from high upper left, pale and cool, lifting the frost on the bark and the rim of his pauldron, "
|
| 37 |
+
"while the forest floor falls into deep blue shadow where texture remains visible. The palette stays "
|
| 38 |
+
"restrained: slate blue, grey-white snow, muted brown leather and tarnished steel, with neutral white balance "
|
| 39 |
+
"and fine film grain."),
|
| 40 |
+
("Anime sword duel", "16:9",
|
| 41 |
+
"A frame from a Japanese anime television series, hand-drawn cel animation over a painted background, "
|
| 42 |
+
"widescreen composition. On a rocky mountaintop in a thunderstorm, a swordsman in a torn dark coat swings a "
|
| 43 |
+
"massive broadsword that glows pale blue, its blade mid-arc and his teeth clenched. Opposite him, a slender "
|
| 44 |
+
"fighter in a white scarf blurs into motion, leaving streaks of afterimage and a ring of cracked stone. "
|
| 45 |
+
"Lightning forks across the sky behind them, rain slants through the frame, and bright white highlights gleam "
|
| 46 |
+
"on wet rock."),
|
| 47 |
+
("1970s sci-fi comic", "3:2",
|
| 48 |
+
"A gouache and ink illustration in the style of a 1970s European science-fiction comic, filling the whole "
|
| 49 |
+
"picture edge to edge. A lone space traveler in a bulky white suit stands on a rocky ledge, helmet visor "
|
| 50 |
+
"catching the faint light, gazing up at towering mushroom trees with broad caps and thick pale stalks. A dim "
|
| 51 |
+
"violet sky stretches above, with a few pale spores drifting through the air. Flat violet and ochre color with "
|
| 52 |
+
"visible brush texture and inked contour lines, shown in a wide landscape view from a low angle."),
|
| 53 |
+
("Linen lookbook", "2:3",
|
| 54 |
+
"A lookbook photograph by an experienced fashion photographer: a woman in her late twenties stands full length "
|
| 55 |
+
"in a pale sandy-linen dress, a loose sleeveless A-line with a square neckline, a gathered waist and a hem that "
|
| 56 |
+
"falls just below the calf. The natural linen creases softly at the hips and lifts slightly as she shifts her "
|
| 57 |
+
"weight onto one foot in a light breeze. Her chestnut hair is loosely tied back, she wears simple leather "
|
| 58 |
+
"sandals and small gold hoops, and her expression is calm as she looks just past the camera. The location is a "
|
| 59 |
+
"whitewashed plaster wall beside a weathered wooden door, with the shadow of a palm frond falling across the "
|
| 60 |
+
"plaster. Late-afternoon sunlight from the left gives warm directional light with visible detail in the "
|
| 61 |
+
"shadows. Shot at 85mm from a few steps back at standing eye height, shallow depth of field, true white "
|
| 62 |
+
"balance, fine grain, restrained palette of cream, sand and sage. No text, no other objects."),
|
| 63 |
+
("Risograph cyclist", "3:4",
|
| 64 |
+
"A two-color risograph print on cream paper, the artwork running edge to edge with no border or margin. A "
|
| 65 |
+
"cyclist in a hooded jacket and helmet leans into a turn on a bicycle down a wide city avenue, seen from a low "
|
| 66 |
+
"side angle. Behind them rise tall apartment blocks with rows of windows, a tram line curving into the "
|
| 67 |
+
"distance, and a few parked cars. Only the cyclist, the buildings and the road are shown. The image uses "
|
| 68 |
+
"fluorescent pink and deep blue inks, which overprint to a dark purple where they overlap. Visible halftone "
|
| 69 |
+
"dots, grainy ink coverage, and slight misregistration between the two passes run across the whole sheet."),
|
| 70 |
+
("Byzantine mosaic", "3:4",
|
| 71 |
+
"A Byzantine gold-ground mosaic panel, small glass and stone tesserae set in lime mortar, with the gold tiles "
|
| 72 |
+
"catching light unevenly. A lighthouse keeper in a dark blue tunic and short cloak stands at the base of a tall "
|
| 73 |
+
"white tower, one hand raised to shield his eyes as he looks out to sea. Behind him a lantern glows with a "
|
| 74 |
+
"red-gold flame, and the water below is built in bands of teal and deep blue tesserae, with a small ship on the "
|
| 75 |
+
"horizon. The gold ground fills the background, the figures are outlined in dark grey-brown tesserae, and the "
|
| 76 |
+
"grout lines are slightly uneven. The mosaic fills the whole image edge to edge, with no frame or border."),
|
| 77 |
+
("Kirchner street at night", "3:4",
|
| 78 |
+
"A German Expressionist oil painting on canvas in the manner of Ernst Ludwig Kirchner's Berlin street scenes. A "
|
| 79 |
+
"narrow city street at night tilts sharply toward the viewer, its buildings leaning into angular, flattened "
|
| 80 |
+
"planes. Gaunt figures in dark coats hurry past a gas lamp whose light is a thick yellow smear dragged across "
|
| 81 |
+
"the wet paving. Heavy impasto brushstrokes and visible palette-knife edges, with raw cadmium yellow, cobalt "
|
| 82 |
+
"blue and vermilion patches in the lit windows. The paint covers the canvas edge to edge, with no frame or "
|
| 83 |
+
"border."),
|
| 84 |
+
]
|
p_h3_image_fast.py
ADDED
|
@@ -0,0 +1,857 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Kernel-level speedups for p-h3-image (the diffusers fork) on an RTX PRO 6000 Blackwell (sm_120). No torch.compile.
|
| 2 |
+
|
| 3 |
+
* `install_adaln_tables`: every block's AdaLN projection (13B of the transformer's 33B parameters) depends only on
|
| 4 |
+
the timestep, and the pipeline only ever uses the timesteps of its fixed sigma grid. Their outputs are computed once
|
| 5 |
+
per transformer call signature and the projection weights are dropped. Bitwise identical to the projections.
|
| 6 |
+
* `fuse_qkv`: one GEMM for q, k and v.
|
| 7 |
+
* `quantize_fp8`: W8A8 linears, per-output-channel weight scales and per-row dynamic activation scales, through
|
| 8 |
+
torch._scaled_mm (cuBLASLt FP8). Activations are quantized by one Triton kernel per GEMM input.
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
from __future__ import annotations
|
| 12 |
+
|
| 13 |
+
import inspect
|
| 14 |
+
import time
|
| 15 |
+
|
| 16 |
+
import torch
|
| 17 |
+
import triton
|
| 18 |
+
import triton.language as tl
|
| 19 |
+
from torch import nn
|
| 20 |
+
|
| 21 |
+
FP8 = torch.float8_e4m3fn
|
| 22 |
+
FP8_MAX = 448.0
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def _free_cache() -> None:
|
| 26 |
+
try:
|
| 27 |
+
if torch.cuda.is_available() and torch.cuda.is_initialized():
|
| 28 |
+
torch.cuda.empty_cache()
|
| 29 |
+
except Exception: # noqa: BLE001 - ZeroGPU emulates CUDA outside GPU calls
|
| 30 |
+
pass
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
# AdaLN tables -------------------------------------------------------------------------------------------------------
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
class AdaLNTable(nn.Module):
|
| 37 |
+
"""Replaces `MiniMaxH3AdaLayerNormModulation`: returns its precomputed output for the current call's timesteps."""
|
| 38 |
+
|
| 39 |
+
def __init__(self, tables: list[torch.Tensor], hidden_size: int, lookup: dict):
|
| 40 |
+
super().__init__()
|
| 41 |
+
for index, table in enumerate(tables):
|
| 42 |
+
self.register_buffer(f"table_{index}", table, persistent=False)
|
| 43 |
+
self.hidden_size = hidden_size
|
| 44 |
+
self.lookup = lookup
|
| 45 |
+
|
| 46 |
+
def forward(self, temb: torch.Tensor) -> tuple[torch.Tensor, ...]:
|
| 47 |
+
table = getattr(self, f"table_{self.lookup['index']}")
|
| 48 |
+
return table.view(-1, 6 * self.hidden_size).chunk(6, dim=-1)
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
@torch.no_grad()
|
| 52 |
+
def install_adaln_tables(transformer, calls: list[torch.Tensor]) -> int:
|
| 53 |
+
"""Precompute every block's AdaLN output for each distinct `timestep` argument in `calls` (as the pipeline passes
|
| 54 |
+
them, one tensor per transformer call) and drop the projection weights. Each table is computed with the call's own
|
| 55 |
+
shapes, exactly as the forward would, so the outputs are bitwise identical."""
|
| 56 |
+
device = next(transformer.parameters()).device
|
| 57 |
+
keys: dict[tuple, int] = {}
|
| 58 |
+
tembs = []
|
| 59 |
+
for timestep in calls:
|
| 60 |
+
key = tuple(timestep.tolist())
|
| 61 |
+
if key not in keys:
|
| 62 |
+
keys[key] = len(keys)
|
| 63 |
+
temb = transformer.time_proj(timestep.to(device))
|
| 64 |
+
tembs.append(transformer.time_embedder(temb.to(transformer.time_embedder.linear_1.weight.dtype)))
|
| 65 |
+
lookup = {"keys": keys, "index": None}
|
| 66 |
+
freed = 0
|
| 67 |
+
for block in transformer.transformer_blocks:
|
| 68 |
+
proj = block.adaln_proj
|
| 69 |
+
dtype = proj.linear.weight.dtype
|
| 70 |
+
tables = [proj.linear(nn.functional.silu(temb).to(dtype)).contiguous() for temb in tembs]
|
| 71 |
+
freed += sum(p.numel() for p in proj.parameters())
|
| 72 |
+
block.adaln_proj = AdaLNTable(tables, proj.hidden_size, lookup)
|
| 73 |
+
del proj
|
| 74 |
+
|
| 75 |
+
def set_index(module, args, kwargs):
|
| 76 |
+
key = tuple(kwargs["timestep"].tolist())
|
| 77 |
+
if key not in keys:
|
| 78 |
+
raise ValueError(f"No AdaLN table for timesteps {key}; known: {list(keys)}")
|
| 79 |
+
lookup["index"] = keys[key]
|
| 80 |
+
|
| 81 |
+
transformer.register_forward_pre_hook(set_index, with_kwargs=True)
|
| 82 |
+
_free_cache()
|
| 83 |
+
return freed
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
# QKV fusion ---------------------------------------------------------------------------------------------------------
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
@torch.no_grad()
|
| 90 |
+
def fuse_qkv(transformer) -> None:
|
| 91 |
+
for block in transformer.transformer_blocks:
|
| 92 |
+
attn = block.attn
|
| 93 |
+
weight = torch.cat([attn.to_q.weight, attn.to_k.weight, attn.to_v.weight])
|
| 94 |
+
attn.to_qkv = nn.Linear(weight.shape[1], weight.shape[0], bias=False, device=weight.device, dtype=weight.dtype)
|
| 95 |
+
attn.to_qkv.weight.copy_(weight)
|
| 96 |
+
del attn.to_q, attn.to_k, attn.to_v
|
| 97 |
+
attn.fused_projections = True
|
| 98 |
+
_free_cache()
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
# FP8 W8A8 -----------------------------------------------------------------------------------------------------------
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
@triton.jit
|
| 105 |
+
def _rowwise_fp8_kernel(x_ptr, q_ptr, s_ptr, K, stride_x, stride_q, BLOCK: tl.constexpr):
|
| 106 |
+
row = tl.program_id(0)
|
| 107 |
+
offs = tl.arange(0, BLOCK)
|
| 108 |
+
mask = offs < K
|
| 109 |
+
x = tl.load(x_ptr + row * stride_x + offs, mask=mask, other=0.0).to(tl.float32)
|
| 110 |
+
amax = tl.maximum(tl.max(tl.abs(x), axis=0), 1e-12)
|
| 111 |
+
tl.store(q_ptr + row * stride_q + offs, (x * (448.0 / amax)).to(tl.float8e4nv), mask=mask)
|
| 112 |
+
tl.store(s_ptr + row, amax / 448.0)
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
def rowwise_fp8(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
| 116 |
+
"""(M, K) bf16 -> (M, K) e4m3 and (M, 1) float32 scales with x ~= q * scale."""
|
| 117 |
+
M, K = x.shape
|
| 118 |
+
q = torch.empty((M, K), device=x.device, dtype=FP8)
|
| 119 |
+
scale = torch.empty((M, 1), device=x.device, dtype=torch.float32)
|
| 120 |
+
block = triton.next_power_of_2(K)
|
| 121 |
+
_rowwise_fp8_kernel[(M,)](x, q, scale, K, x.stride(0), q.stride(0), BLOCK=block, num_warps=8 if block <= 8192 else 16)
|
| 122 |
+
return q, scale
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
_W8A16_CONFIGS = [triton.Config({"BLOCK_N": n, "BLOCK_K": k}, num_warps=w, num_stages=4)
|
| 126 |
+
for n in (16, 32, 64) for k in (128, 256) for w in (4, 8)]
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
# Tuned configs persist in Triton's cache dir, so a new process (ZeroGPU workers) skips the autotune.
|
| 130 |
+
_AUTOTUNE_CACHE = {"cache_results": True} if "cache_results" in inspect.signature(triton.autotune).parameters else {}
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
@triton.autotune(configs=_W8A16_CONFIGS, key=["N", "K"], **_AUTOTUNE_CACHE)
|
| 134 |
+
@triton.jit
|
| 135 |
+
def _w8a16_kernel(x_ptr, w_ptr, s_ptr, b_ptr, y_ptr, M, N, K, stride_x, stride_w, stride_y,
|
| 136 |
+
HAS_BIAS: tl.constexpr, BLOCK_N: tl.constexpr, BLOCK_K: tl.constexpr):
|
| 137 |
+
offs_n = tl.program_id(0) * BLOCK_N + tl.arange(0, BLOCK_N)
|
| 138 |
+
offs_m = tl.arange(0, 16)
|
| 139 |
+
acc = tl.zeros((16, BLOCK_N), dtype=tl.float32)
|
| 140 |
+
for k0 in range(0, K, BLOCK_K):
|
| 141 |
+
offs_k = k0 + tl.arange(0, BLOCK_K)
|
| 142 |
+
w = tl.load(w_ptr + offs_n[:, None] * stride_w + offs_k[None, :],
|
| 143 |
+
mask=(offs_n[:, None] < N) & (offs_k[None, :] < K), other=0.0).to(tl.bfloat16)
|
| 144 |
+
x = tl.load(x_ptr + offs_m[:, None] * stride_x + offs_k[None, :],
|
| 145 |
+
mask=(offs_m[:, None] < M) & (offs_k[None, :] < K), other=0.0)
|
| 146 |
+
acc += tl.dot(x, tl.trans(w))
|
| 147 |
+
acc *= tl.load(s_ptr + offs_n, mask=offs_n < N, other=0.0)[None, :]
|
| 148 |
+
if HAS_BIAS:
|
| 149 |
+
acc += tl.load(b_ptr + offs_n, mask=offs_n < N, other=0.0).to(tl.float32)[None, :]
|
| 150 |
+
tl.store(y_ptr + offs_m[:, None] * stride_y + offs_n[None, :], acc.to(y_ptr.dtype.element_ty),
|
| 151 |
+
mask=(offs_m[:, None] < M) & (offs_n[None, :] < N))
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
def w8a16_gemv(x: torch.Tensor, weight: torch.Tensor, scale: torch.Tensor, bias: torch.Tensor | None) -> torch.Tensor:
|
| 155 |
+
"""y = x @ (weight * scale).T for at most 16 rows of x: FP8 weights dequantized in registers, bandwidth-bound."""
|
| 156 |
+
M, K = x.shape
|
| 157 |
+
N = weight.shape[0]
|
| 158 |
+
x = x.contiguous()
|
| 159 |
+
y = torch.empty((M, N), device=x.device, dtype=x.dtype)
|
| 160 |
+
grid = lambda meta: (triton.cdiv(N, meta["BLOCK_N"]),)
|
| 161 |
+
_w8a16_kernel[grid](x, weight, scale, bias if bias is not None else scale, y, M, N, K, x.stride(0),
|
| 162 |
+
weight.stride(0), y.stride(0), HAS_BIAS=bias is not None)
|
| 163 |
+
return y
|
| 164 |
+
|
| 165 |
+
|
| 166 |
+
class FP8Linear(nn.Module):
|
| 167 |
+
def __init__(self, linear: nn.Linear, fast_accum: bool = False):
|
| 168 |
+
super().__init__()
|
| 169 |
+
weight = linear.weight.detach()
|
| 170 |
+
scale = weight.float().abs().amax(dim=1, keepdim=True).clamp_min(1e-12) / FP8_MAX
|
| 171 |
+
self.register_buffer("weight", (weight.float() / scale).to(FP8).contiguous()) # (N, K)
|
| 172 |
+
self.register_buffer("weight_scale", scale.t().contiguous()) # (1, N)
|
| 173 |
+
self.bias = linear.bias
|
| 174 |
+
self.in_features, self.out_features = linear.in_features, linear.out_features
|
| 175 |
+
self.out_dtype = weight.dtype
|
| 176 |
+
self.fast_accum = fast_accum
|
| 177 |
+
|
| 178 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 179 |
+
shape = x.shape
|
| 180 |
+
rows = x.reshape(-1, shape[-1])
|
| 181 |
+
if rows.shape[0] <= 16: # decode: weight-only, the GEMM is a bandwidth-bound GEMV
|
| 182 |
+
return w8a16_gemv(rows.to(self.out_dtype), self.weight, self.weight_scale.view(-1),
|
| 183 |
+
self.bias).view(*shape[:-1], self.out_features)
|
| 184 |
+
q, scale = rowwise_fp8(rows)
|
| 185 |
+
out = torch._scaled_mm(q, self.weight.t(), scale_a=scale, scale_b=self.weight_scale, bias=self.bias,
|
| 186 |
+
out_dtype=self.out_dtype, use_fast_accum=self.fast_accum)
|
| 187 |
+
return out.view(*shape[:-1], self.out_features)
|
| 188 |
+
|
| 189 |
+
|
| 190 |
+
@torch.no_grad()
|
| 191 |
+
def quantize_fp8(module: nn.Module, fast_accum: bool = False, skip: tuple[str, ...] = ()) -> int:
|
| 192 |
+
"""Replace every nn.Linear under `module` (names containing an entry of `skip` excluded) by an FP8Linear."""
|
| 193 |
+
count = 0
|
| 194 |
+
for name, child in list(module.named_modules()):
|
| 195 |
+
for child_name, leaf in list(child.named_children()):
|
| 196 |
+
full = f"{name}.{child_name}" if name else child_name
|
| 197 |
+
if isinstance(leaf, nn.Linear) and not any(s in full for s in skip):
|
| 198 |
+
setattr(child, child_name, FP8Linear(leaf, fast_accum))
|
| 199 |
+
del leaf
|
| 200 |
+
count += 1
|
| 201 |
+
_free_cache()
|
| 202 |
+
return count
|
| 203 |
+
|
| 204 |
+
|
| 205 |
+
# CUDA-graph greedy decoding -----------------------------------------------------------------------------------------
|
| 206 |
+
|
| 207 |
+
|
| 208 |
+
class GraphedGreedyGenerate:
|
| 209 |
+
"""Greedy decoding of one sequence for a Qwen3_5ForConditionalGeneration, with the decode step replayed as a CUDA
|
| 210 |
+
graph. A drop-in for `model.generate(**inputs, max_new_tokens=..., do_sample=False, eos_token_id=...)`.
|
| 211 |
+
|
| 212 |
+
The prefill runs eagerly into a persistent StaticCache (its linear-attention states and KV tensors are updated in
|
| 213 |
+
place, so their addresses never change). One decode step (embed, 32 layers, lm_head, argmax) is captured once over
|
| 214 |
+
static buffers: the input token, its absolute position, the M-RoPE offset and the attention mask over the cache.
|
| 215 |
+
Per token only the graph is replayed; the end token is checked every `check_every` tokens.
|
| 216 |
+
"""
|
| 217 |
+
|
| 218 |
+
def __init__(self, model, max_cache_len: int = 12288, max_new_tokens: int = 4096, check_every: int = 16):
|
| 219 |
+
from transformers import StaticCache
|
| 220 |
+
|
| 221 |
+
self.model, self.generate_eager = model, model.generate
|
| 222 |
+
self.max_cache_len, self.check_every = max_cache_len, check_every
|
| 223 |
+
device = model.device
|
| 224 |
+
self.cache = StaticCache(config=model.config, max_cache_len=max_cache_len)
|
| 225 |
+
self.tok = torch.zeros((1, 1), dtype=torch.long, device=device)
|
| 226 |
+
self.pos = torch.zeros((1,), dtype=torch.long, device=device)
|
| 227 |
+
self.offset = torch.zeros((4, 1, 1), dtype=torch.long, device=device)
|
| 228 |
+
self.mask = torch.zeros((1, 1, 1, max_cache_len), dtype=torch.bool, device=device)
|
| 229 |
+
self.out = torch.zeros((max_new_tokens,), dtype=torch.long, device=device)
|
| 230 |
+
self.count = torch.zeros((1,), dtype=torch.long, device=device)
|
| 231 |
+
self.graph = None
|
| 232 |
+
|
| 233 |
+
def _step(self):
|
| 234 |
+
text_model = self.model.model.language_model
|
| 235 |
+
self.mask.index_fill_(3, self.pos, True)
|
| 236 |
+
hidden = text_model(
|
| 237 |
+
inputs_embeds=text_model.embed_tokens(self.tok),
|
| 238 |
+
position_ids=self.pos.view(1, 1, 1) + self.offset,
|
| 239 |
+
attention_mask={"full_attention": self.mask, "linear_attention": None},
|
| 240 |
+
past_key_values=self.cache,
|
| 241 |
+
use_cache=True,
|
| 242 |
+
).last_hidden_state
|
| 243 |
+
token = self.model.lm_head(hidden[:, -1]).argmax(-1)
|
| 244 |
+
self.out.index_copy_(0, self.count, token)
|
| 245 |
+
self.tok.copy_(token.view(1, 1))
|
| 246 |
+
self.pos.add_(1)
|
| 247 |
+
self.count.add_(1)
|
| 248 |
+
|
| 249 |
+
def _prefill(self, input_ids, kwargs):
|
| 250 |
+
self.cache.reset()
|
| 251 |
+
self.mask.zero_()
|
| 252 |
+
self.model.model.rope_deltas = None
|
| 253 |
+
logits = self.model(input_ids=input_ids, past_key_values=self.cache, use_cache=True, logits_to_keep=1,
|
| 254 |
+
**kwargs).logits
|
| 255 |
+
token = logits[:, -1].argmax(-1)
|
| 256 |
+
rope_deltas = self.model.model.rope_deltas
|
| 257 |
+
self.offset.zero_()
|
| 258 |
+
if rope_deltas is not None:
|
| 259 |
+
self.offset[1:] = rope_deltas.view(-1)[0]
|
| 260 |
+
prompt_length = input_ids.shape[1]
|
| 261 |
+
self.mask[..., :prompt_length] = True
|
| 262 |
+
self.out[0] = token[0]
|
| 263 |
+
self.count.fill_(1)
|
| 264 |
+
self.tok.copy_(token.view(1, 1))
|
| 265 |
+
self.pos.fill_(prompt_length)
|
| 266 |
+
|
| 267 |
+
def _capture(self):
|
| 268 |
+
stream = torch.cuda.Stream()
|
| 269 |
+
stream.wait_stream(torch.cuda.current_stream())
|
| 270 |
+
with torch.cuda.stream(stream):
|
| 271 |
+
for _ in range(3):
|
| 272 |
+
self._step()
|
| 273 |
+
torch.cuda.current_stream().wait_stream(stream)
|
| 274 |
+
self.graph = torch.cuda.CUDAGraph()
|
| 275 |
+
with torch.cuda.graph(self.graph):
|
| 276 |
+
self._step()
|
| 277 |
+
|
| 278 |
+
@torch.no_grad()
|
| 279 |
+
def generate(self, input_ids, max_new_tokens: int, eos_token_id: int, **kwargs):
|
| 280 |
+
kwargs.pop("do_sample", None)
|
| 281 |
+
kwargs.pop("pad_token_id", None)
|
| 282 |
+
if input_ids.shape[0] != 1 or input_ids.shape[1] + max_new_tokens > self.max_cache_len:
|
| 283 |
+
return self.generate_eager(input_ids=input_ids, max_new_tokens=max_new_tokens, do_sample=False,
|
| 284 |
+
eos_token_id=eos_token_id, **kwargs)
|
| 285 |
+
if self.graph is None:
|
| 286 |
+
self._prefill(input_ids, kwargs)
|
| 287 |
+
self._capture()
|
| 288 |
+
self._prefill(input_ids, kwargs)
|
| 289 |
+
generated = 1
|
| 290 |
+
while generated < max_new_tokens and not bool((self.out[:generated] == eos_token_id).any()):
|
| 291 |
+
for _ in range(min(self.check_every, max_new_tokens - generated)):
|
| 292 |
+
self.graph.replay()
|
| 293 |
+
generated = min(generated + self.check_every, max_new_tokens)
|
| 294 |
+
tokens = self.out[:generated]
|
| 295 |
+
ends = (tokens == eos_token_id).nonzero()
|
| 296 |
+
if ends.numel():
|
| 297 |
+
tokens = tokens[: int(ends[0, 0]) + 1]
|
| 298 |
+
return torch.cat([input_ids, tokens[None].to(input_ids.device)], dim=1)
|
| 299 |
+
|
| 300 |
+
|
| 301 |
+
# MTP speculative decoding -------------------------------------------------------------------------------------------
|
| 302 |
+
|
| 303 |
+
|
| 304 |
+
class MTPHead(nn.Module):
|
| 305 |
+
"""Qwen3.5's multi-token-prediction head: (hidden state at i, embedding of token i+1) -> hidden for token i+2."""
|
| 306 |
+
|
| 307 |
+
def __init__(self, config):
|
| 308 |
+
super().__init__()
|
| 309 |
+
from transformers.models.qwen3_5.modeling_qwen3_5 import Qwen3_5DecoderLayer, Qwen3_5RMSNorm
|
| 310 |
+
|
| 311 |
+
size, eps = config.hidden_size, config.rms_norm_eps
|
| 312 |
+
self.config = config
|
| 313 |
+
self.fc = nn.Linear(2 * size, size, bias=False)
|
| 314 |
+
self.pre_fc_norm_embedding = Qwen3_5RMSNorm(size, eps)
|
| 315 |
+
self.pre_fc_norm_hidden = Qwen3_5RMSNorm(size, eps)
|
| 316 |
+
self.layers = nn.ModuleList([Qwen3_5DecoderLayer(config, 0)])
|
| 317 |
+
self.norm = Qwen3_5RMSNorm(size, eps)
|
| 318 |
+
|
| 319 |
+
def forward(self, embeds, hidden, position_embeddings, text_positions, mask, cache):
|
| 320 |
+
x = self.fc(torch.cat([self.pre_fc_norm_embedding(embeds), self.pre_fc_norm_hidden(hidden)], dim=-1))
|
| 321 |
+
x = self.layers[0](x, position_embeddings=position_embeddings, attention_mask=mask, position_ids=text_positions,
|
| 322 |
+
past_key_values=cache, use_cache=True)
|
| 323 |
+
return self.norm(x)
|
| 324 |
+
|
| 325 |
+
|
| 326 |
+
MTP_REPO, MTP_REVISION = "Qwen/Qwen3.5-9B", "c202236235762e1c871ad0ccb60c8ee5ba337b9a"
|
| 327 |
+
|
| 328 |
+
|
| 329 |
+
def read_hub_tensors(repo: str, revision: str, prefix: str) -> dict[str, torch.Tensor]:
|
| 330 |
+
"""The tensors named `prefix*` of a sharded safetensors repo, read by byte range (no full shard downloads)."""
|
| 331 |
+
import json
|
| 332 |
+
import struct
|
| 333 |
+
|
| 334 |
+
from huggingface_hub import HfFileSystem, hf_hub_download
|
| 335 |
+
|
| 336 |
+
dtypes = {"BF16": torch.bfloat16, "F16": torch.float16, "F32": torch.float32}
|
| 337 |
+
index = json.load(open(hf_hub_download(repo, "model.safetensors.index.json", revision=revision)))
|
| 338 |
+
fs, tensors = HfFileSystem(), {}
|
| 339 |
+
for file in sorted({f for name, f in index["weight_map"].items() if name.startswith(prefix)}):
|
| 340 |
+
with fs.open(f"{repo}/{file}", "rb", revision=revision) as handle:
|
| 341 |
+
header_size = struct.unpack("<Q", handle.read(8))[0]
|
| 342 |
+
header = json.loads(handle.read(header_size))
|
| 343 |
+
for name, meta in header.items():
|
| 344 |
+
if name.startswith(prefix):
|
| 345 |
+
start, end = meta["data_offsets"]
|
| 346 |
+
handle.seek(8 + header_size + start)
|
| 347 |
+
data = bytearray(handle.read(end - start))
|
| 348 |
+
tensors[name] = torch.frombuffer(data, dtype=dtypes[meta["dtype"]]).reshape(meta["shape"])
|
| 349 |
+
return tensors
|
| 350 |
+
|
| 351 |
+
|
| 352 |
+
def load_mtp_head(model, repo: str = MTP_REPO, revision: str = MTP_REVISION) -> MTPHead:
|
| 353 |
+
"""Qwen3.5-9B's MTP head (transformers drops it), used as the prompt upsampler's draft model."""
|
| 354 |
+
import copy
|
| 355 |
+
|
| 356 |
+
config = copy.deepcopy(model.config.get_text_config())
|
| 357 |
+
config.num_hidden_layers, config.layer_types = 1, ["full_attention"]
|
| 358 |
+
state = {name[len("mtp."):]: tensor for name, tensor in read_hub_tensors(repo, revision, "mtp.").items()}
|
| 359 |
+
head = MTPHead(config)
|
| 360 |
+
head.load_state_dict(state)
|
| 361 |
+
return head.to(model.device, model.dtype).eval()
|
| 362 |
+
|
| 363 |
+
|
| 364 |
+
GDN_VERIFY = {"torch_reference": False}
|
| 365 |
+
|
| 366 |
+
|
| 367 |
+
def _gdn_verify(module, hidden_states, cache_params):
|
| 368 |
+
"""Qwen3_5GatedDeltaNet over a few cached tokens, one token at a time, keeping the conv and recurrent states after
|
| 369 |
+
every token in `module._spec_snaps` so a rejected draft rolls back by index."""
|
| 370 |
+
from torch.nn import functional as F
|
| 371 |
+
from transformers.models.qwen3_5 import modeling_qwen3_5 as qm
|
| 372 |
+
|
| 373 |
+
batch, length, _ = hidden_states.shape
|
| 374 |
+
layer = cache_params.layers[module.layer_idx]
|
| 375 |
+
conv_state, recurrent_state = layer.conv_states[0], layer.recurrent_states[0]
|
| 376 |
+
conv_snaps, recurrent_snaps = module._spec_snaps
|
| 377 |
+
mixed = module.in_proj_qkv(hidden_states).transpose(1, 2)
|
| 378 |
+
z = module.in_proj_z(hidden_states).reshape(batch, length, -1, module.head_v_dim)
|
| 379 |
+
b, a = module.in_proj_b(hidden_states), module.in_proj_a(hidden_states)
|
| 380 |
+
convolved = []
|
| 381 |
+
for j in range(length):
|
| 382 |
+
convolved.append(qm.causal_conv1d_update(mixed[:, :, j : j + 1], conv_state, module.conv1d.weight.squeeze(1),
|
| 383 |
+
module.conv1d.bias, module.activation))
|
| 384 |
+
conv_snaps[j].copy_(conv_state)
|
| 385 |
+
mixed = torch.cat(convolved, dim=2).transpose(1, 2)
|
| 386 |
+
query, key, value = torch.split(mixed, [module.key_dim, module.key_dim, module.value_dim], dim=-1)
|
| 387 |
+
query = query.reshape(batch, length, -1, module.head_k_dim)
|
| 388 |
+
key = key.reshape(batch, length, -1, module.head_k_dim)
|
| 389 |
+
value = value.reshape(batch, length, -1, module.head_v_dim)
|
| 390 |
+
beta = b.sigmoid()
|
| 391 |
+
g = -module.A_log.float().exp() * F.softplus(a.float() + module.dt_bias)
|
| 392 |
+
if module.num_v_heads // module.num_k_heads > 1:
|
| 393 |
+
query = query.repeat_interleave(module.num_v_heads // module.num_k_heads, dim=2)
|
| 394 |
+
key = key.repeat_interleave(module.num_v_heads // module.num_k_heads, dim=2)
|
| 395 |
+
recurrence = qm.torch_recurrent_gated_delta_rule
|
| 396 |
+
if GDN_VERIFY["torch_reference"]:
|
| 397 |
+
recurrence = getattr(recurrence, "__wrapped__", recurrence)
|
| 398 |
+
outputs = []
|
| 399 |
+
for j in range(length):
|
| 400 |
+
step = [t[:, j : j + 1].contiguous() for t in (query, key, value, g, beta)]
|
| 401 |
+
out, state = recurrence(step[0], step[1], step[2], g=step[3], beta=step[4],
|
| 402 |
+
initial_state=recurrent_state.clone(), output_final_state=True,
|
| 403 |
+
use_qk_l2norm_in_kernel=True)
|
| 404 |
+
recurrent_state.copy_(state)
|
| 405 |
+
recurrent_snaps[j].copy_(recurrent_state)
|
| 406 |
+
outputs.append(out)
|
| 407 |
+
core = module.norm(torch.cat(outputs, dim=1).reshape(-1, module.head_v_dim), z.reshape(-1, module.head_v_dim))
|
| 408 |
+
return module.out_proj(core.reshape(batch, length, -1))
|
| 409 |
+
|
| 410 |
+
|
| 411 |
+
def _install_gdn_verify(module) -> None:
|
| 412 |
+
"""Route `module`'s forward through `_gdn_verify` while `module._spec_verify` is set. Patched on the instance:
|
| 413 |
+
`set_use_kernels` installs instance-level forwards, which a class-level patch would not reach."""
|
| 414 |
+
if getattr(module, "_spec_patched", False):
|
| 415 |
+
return
|
| 416 |
+
original = module.forward
|
| 417 |
+
|
| 418 |
+
def forward(hidden_states, cache_params=None, attention_mask=None, **kwargs):
|
| 419 |
+
if module._spec_verify:
|
| 420 |
+
return _gdn_verify(module, hidden_states, cache_params)
|
| 421 |
+
return original(hidden_states, cache_params=cache_params, attention_mask=attention_mask, **kwargs)
|
| 422 |
+
|
| 423 |
+
module.forward, module._spec_verify, module._spec_patched = forward, False, True
|
| 424 |
+
|
| 425 |
+
|
| 426 |
+
class MTPGreedyGenerate:
|
| 427 |
+
"""Greedy decoding of one sequence with one MTP draft token per step, each step one CUDA graph. Lossless: every
|
| 428 |
+
emitted token is the main model's own greedy choice; an accepted draft adds a second token to the step.
|
| 429 |
+
|
| 430 |
+
A step feeds `[t, draft]` to the main model (its Gated DeltaNet layers keep their states after each token, the
|
| 431 |
+
full-attention layers write both K/V rows), takes the greedy predictions p0, p1, accepts the draft if it equals
|
| 432 |
+
p0, rolls the states back to the accepted prefix and emits `draft, p1` or `p0`. The MTP head then runs on the two
|
| 433 |
+
(hidden, next token) pairs of the step and drafts from the accepted one. The end token is checked every
|
| 434 |
+
`check_every` steps.
|
| 435 |
+
"""
|
| 436 |
+
|
| 437 |
+
def __init__(self, model, mtp: MTPHead, max_cache_len: int = 12288, max_new_tokens: int = 4096,
|
| 438 |
+
check_every: int = 8):
|
| 439 |
+
from transformers import StaticCache
|
| 440 |
+
|
| 441 |
+
self.model, self.mtp, self.generate_eager = model, mtp, model.generate
|
| 442 |
+
self.text_model = model.model.language_model
|
| 443 |
+
self.max_cache_len, self.check_every = max_cache_len, check_every
|
| 444 |
+
device = model.device
|
| 445 |
+
self.cache = StaticCache(config=model.config, max_cache_len=max_cache_len)
|
| 446 |
+
self.mtp_cache = StaticCache(config=mtp.config, max_cache_len=max_cache_len)
|
| 447 |
+
self.gdn = [layer.linear_attn for layer in self.text_model.layers if layer.block_type == "linear_attention"]
|
| 448 |
+
for module in self.gdn:
|
| 449 |
+
_install_gdn_verify(module)
|
| 450 |
+
long = dict(dtype=torch.long, device=device)
|
| 451 |
+
self.tok, self.draft, self.pos = (torch.zeros((1,), **long) for _ in range(3))
|
| 452 |
+
self.offset = torch.zeros((4, 1, 1), **long)
|
| 453 |
+
self.keys = torch.arange(max_cache_len, **long)
|
| 454 |
+
self.pair = torch.arange(2, **long)
|
| 455 |
+
self.out = torch.zeros((max_new_tokens + 2 * check_every + 2,), **long)
|
| 456 |
+
self.count, self.accepted = torch.zeros((1,), **long), torch.zeros((1,), **long)
|
| 457 |
+
self.max_new_tokens = max_new_tokens
|
| 458 |
+
self.graph, self.stats = None, {}
|
| 459 |
+
|
| 460 |
+
def _set_verify(self, on: bool):
|
| 461 |
+
for module in self.gdn:
|
| 462 |
+
module._spec_verify = on
|
| 463 |
+
|
| 464 |
+
def _static_lengths(self):
|
| 465 |
+
from transformers.cache_utils import StaticLayer
|
| 466 |
+
|
| 467 |
+
return [layer.cumulative_length for cache in (self.cache, self.mtp_cache) for layer in cache.layers
|
| 468 |
+
if isinstance(layer, StaticLayer)]
|
| 469 |
+
|
| 470 |
+
def _step(self):
|
| 471 |
+
positions = self.pos + self.pair
|
| 472 |
+
position_ids = positions.view(1, 1, 2) + self.offset
|
| 473 |
+
mask = (self.keys.view(1, -1) <= positions.view(-1, 1)).view(1, 1, 2, -1)
|
| 474 |
+
masks = {"full_attention": mask, "linear_attention": None}
|
| 475 |
+
tokens = torch.cat([self.tok, self.draft]).view(1, 2)
|
| 476 |
+
hidden = self.text_model(inputs_embeds=self.text_model.embed_tokens(tokens), position_ids=position_ids,
|
| 477 |
+
attention_mask=masks, past_key_values=self.cache, use_cache=True).last_hidden_state
|
| 478 |
+
preds = self.model.lm_head(hidden).argmax(-1).view(2)
|
| 479 |
+
accept = (preds[0] == self.draft[0]).long().view(1)
|
| 480 |
+
for module in self.gdn:
|
| 481 |
+
layer = self.cache.layers[module.layer_idx]
|
| 482 |
+
conv_snaps, recurrent_snaps = module._spec_snaps
|
| 483 |
+
layer.conv_states[0].copy_(conv_snaps.index_select(0, accept)[0])
|
| 484 |
+
layer.recurrent_states[0].copy_(recurrent_snaps.index_select(0, accept)[0])
|
| 485 |
+
emitted = torch.stack([torch.where(accept.bool(), self.draft, preds[:1])[0], preds[1]])
|
| 486 |
+
self.out.index_copy_(0, self.count + self.pair, emitted)
|
| 487 |
+
cos_sin = self.text_model.rotary_emb(hidden, position_ids[1:])
|
| 488 |
+
mtp_hidden = self.mtp(self.text_model.embed_tokens(emitted.view(1, 2)), hidden, cos_sin, position_ids[0], mask,
|
| 489 |
+
self.mtp_cache)
|
| 490 |
+
draft = self.model.lm_head(mtp_hidden.index_select(1, accept)).argmax(-1).view(1)
|
| 491 |
+
new_length = (self.pos + 1 + accept).view(())
|
| 492 |
+
for length in self._static_lengths():
|
| 493 |
+
length.copy_(new_length)
|
| 494 |
+
self.tok.copy_(torch.where(accept.bool(), preds[1:], preds[:1]))
|
| 495 |
+
self.draft.copy_(draft)
|
| 496 |
+
self.pos.add_(1 + accept)
|
| 497 |
+
self.count.add_(1 + accept)
|
| 498 |
+
self.accepted.add_(accept)
|
| 499 |
+
|
| 500 |
+
def _prefill(self, input_ids, kwargs):
|
| 501 |
+
self.cache.reset()
|
| 502 |
+
self.mtp_cache.reset()
|
| 503 |
+
self._set_verify(False)
|
| 504 |
+
self.model.model.rope_deltas = None
|
| 505 |
+
seen = {}
|
| 506 |
+
hook = self.text_model.register_forward_pre_hook(
|
| 507 |
+
lambda module, args, kw: seen.update(position_ids=kw.get("position_ids")), with_kwargs=True)
|
| 508 |
+
hidden = self.model.model(input_ids=input_ids, past_key_values=self.cache, use_cache=True,
|
| 509 |
+
**kwargs).last_hidden_state
|
| 510 |
+
hook.remove()
|
| 511 |
+
first = self.model.lm_head(hidden[:, -1]).argmax(-1)
|
| 512 |
+
length = input_ids.shape[1]
|
| 513 |
+
steps = torch.arange(length, device=input_ids.device)
|
| 514 |
+
rope = seen.get("position_ids")
|
| 515 |
+
rope = steps.view(1, 1, -1).expand(3, 1, -1) if rope is None else rope[-3:]
|
| 516 |
+
mask = (self.keys.view(1, -1) <= steps.view(-1, 1)).view(1, 1, length, -1)
|
| 517 |
+
next_ids = torch.cat([input_ids[:, 1:], first.view(1, 1)], dim=1)
|
| 518 |
+
mtp_hidden = self.mtp(self.text_model.embed_tokens(next_ids), hidden, self.text_model.rotary_emb(hidden, rope),
|
| 519 |
+
steps.view(1, -1), mask, self.mtp_cache)
|
| 520 |
+
draft = self.model.lm_head(mtp_hidden[:, -1]).argmax(-1)
|
| 521 |
+
rope_deltas = self.model.model.rope_deltas
|
| 522 |
+
self.offset.zero_()
|
| 523 |
+
if rope_deltas is not None:
|
| 524 |
+
self.offset[1:] = rope_deltas.view(-1)[0]
|
| 525 |
+
self.out[0] = first[0]
|
| 526 |
+
self.count.fill_(1)
|
| 527 |
+
self.accepted.zero_()
|
| 528 |
+
self.tok.copy_(first)
|
| 529 |
+
self.draft.copy_(draft)
|
| 530 |
+
self.pos.fill_(length)
|
| 531 |
+
self._set_verify(True)
|
| 532 |
+
|
| 533 |
+
def _capture(self):
|
| 534 |
+
for module in self.gdn:
|
| 535 |
+
layer = self.cache.layers[module.layer_idx]
|
| 536 |
+
module._spec_snaps = (torch.empty((2, *layer.conv_states[0].shape), dtype=layer.conv_states[0].dtype,
|
| 537 |
+
device=layer.conv_states[0].device),
|
| 538 |
+
torch.empty((2, *layer.recurrent_states[0].shape),
|
| 539 |
+
dtype=layer.recurrent_states[0].dtype,
|
| 540 |
+
device=layer.recurrent_states[0].device))
|
| 541 |
+
stream = torch.cuda.Stream()
|
| 542 |
+
stream.wait_stream(torch.cuda.current_stream())
|
| 543 |
+
with torch.cuda.stream(stream):
|
| 544 |
+
for _ in range(3):
|
| 545 |
+
self._step()
|
| 546 |
+
torch.cuda.current_stream().wait_stream(stream)
|
| 547 |
+
self.graph = torch.cuda.CUDAGraph()
|
| 548 |
+
with torch.cuda.graph(self.graph):
|
| 549 |
+
self._step()
|
| 550 |
+
|
| 551 |
+
@torch.no_grad()
|
| 552 |
+
def generate(self, input_ids, max_new_tokens: int, eos_token_id: int, **kwargs):
|
| 553 |
+
kwargs.pop("do_sample", None)
|
| 554 |
+
kwargs.pop("pad_token_id", None)
|
| 555 |
+
margin = 2 * self.check_every + 2
|
| 556 |
+
self._set_verify(False)
|
| 557 |
+
if (input_ids.shape[0] != 1 or max_new_tokens > self.max_new_tokens
|
| 558 |
+
or input_ids.shape[1] + max_new_tokens + margin > self.max_cache_len):
|
| 559 |
+
return self.generate_eager(input_ids=input_ids, max_new_tokens=max_new_tokens, do_sample=False,
|
| 560 |
+
eos_token_id=eos_token_id, **kwargs)
|
| 561 |
+
if self.graph is None:
|
| 562 |
+
self._prefill(input_ids, kwargs)
|
| 563 |
+
self._capture()
|
| 564 |
+
self._prefill(input_ids, kwargs)
|
| 565 |
+
steps, count = 0, 1
|
| 566 |
+
while count < max_new_tokens and not bool((self.out[:count] == eos_token_id).any()):
|
| 567 |
+
for _ in range(self.check_every):
|
| 568 |
+
self.graph.replay()
|
| 569 |
+
steps += self.check_every
|
| 570 |
+
count = int(self.count)
|
| 571 |
+
tokens = self.out[: min(count, max_new_tokens)]
|
| 572 |
+
ends = (tokens == eos_token_id).nonzero()
|
| 573 |
+
if ends.numel():
|
| 574 |
+
tokens = tokens[: int(ends[0, 0]) + 1]
|
| 575 |
+
self._set_verify(False)
|
| 576 |
+
self.stats = {"steps": steps, "tokens": int(tokens.numel()), "accepted": int(self.accepted),
|
| 577 |
+
"acceptance": round(int(self.accepted) / max(steps, 1), 3)}
|
| 578 |
+
return torch.cat([input_ids, tokens[None].to(input_ids.device)], dim=1)
|
| 579 |
+
|
| 580 |
+
|
| 581 |
+
# Decode attention ---------------------------------------------------------------------------------------------------
|
| 582 |
+
|
| 583 |
+
|
| 584 |
+
def install_decode_attention(*configs) -> None:
|
| 585 |
+
"""Full attention for a few query rows over a long static cache as two grouped batched GEMVs (K/V read once per
|
| 586 |
+
KV head, no repeat). SDPA's memory-efficient kernel parallelizes a single query poorly. Longer queries (prefill)
|
| 587 |
+
keep SDPA."""
|
| 588 |
+
from transformers import AttentionInterface
|
| 589 |
+
from transformers.integrations.sdpa_attention import sdpa_attention_forward
|
| 590 |
+
|
| 591 |
+
def attention(module, query, key, value, attention_mask, dropout=0.0, scaling=None, **kwargs):
|
| 592 |
+
batch, heads, length, dim = query.shape
|
| 593 |
+
if length > 16 or attention_mask is None or attention_mask.dtype != torch.bool:
|
| 594 |
+
return sdpa_attention_forward(module, query, key, value, attention_mask, dropout=dropout,
|
| 595 |
+
scaling=scaling, **kwargs)
|
| 596 |
+
kv_heads, keys = key.shape[1], key.shape[2]
|
| 597 |
+
groups = heads // kv_heads
|
| 598 |
+
q = query.reshape(batch, kv_heads, groups * length, dim)
|
| 599 |
+
scores = torch.matmul(q, key.transpose(-1, -2)).float().view(batch, kv_heads, groups, length, keys)
|
| 600 |
+
scores = scores * (scaling if scaling is not None else dim**-0.5)
|
| 601 |
+
scores = scores.masked_fill(~attention_mask[..., :keys].view(1, 1, 1, length, keys), float("-inf"))
|
| 602 |
+
probs = scores.softmax(-1).to(value.dtype).view(batch, kv_heads, groups * length, keys)
|
| 603 |
+
out = torch.matmul(probs, value).view(batch, heads, length, -1)
|
| 604 |
+
return out.transpose(1, 2).contiguous(), None
|
| 605 |
+
|
| 606 |
+
AttentionInterface.register("p_h3_decode", attention)
|
| 607 |
+
for config in configs:
|
| 608 |
+
config._attn_implementation = "p_h3_decode"
|
| 609 |
+
|
| 610 |
+
|
| 611 |
+
# Batched VAE tile decode --------------------------------------------------------------------------------------------
|
| 612 |
+
|
| 613 |
+
|
| 614 |
+
def batch_vae_tiles(vae, max_tiles: int = 96) -> None:
|
| 615 |
+
"""Decode the spatial tiles of `AutoencoderKLPH3Image` in batches of up to `max_tiles` instead of one decoder call
|
| 616 |
+
per tile. The tile layout, the per-tile decoder math and the stitching are unchanged."""
|
| 617 |
+
|
| 618 |
+
def decode_clip(z: torch.Tensor) -> torch.Tensor:
|
| 619 |
+
if not vae.use_tiling:
|
| 620 |
+
return vae.decoder(vae.post_quant_conv(z))
|
| 621 |
+
ratio = vae.spatial_compression_ratio
|
| 622 |
+
y_indices, y_lengths, y_overlaps = vae._split_tiles(
|
| 623 |
+
z.shape[-2] * ratio, vae.tile_sample_min_height, vae.tile_sample_min_overlap_height)
|
| 624 |
+
x_indices, x_lengths, x_overlaps = vae._split_tiles(
|
| 625 |
+
z.shape[-1] * ratio, vae.tile_sample_min_width, vae.tile_sample_min_overlap_width)
|
| 626 |
+
tiles = {}
|
| 627 |
+
for i, (i_pos, i_len) in enumerate(zip(y_indices, y_lengths)):
|
| 628 |
+
for j, (j_pos, j_len) in enumerate(zip(x_indices, x_lengths)):
|
| 629 |
+
tiles[(i, j)] = z[..., i_pos // ratio: i_pos // ratio + i_len // ratio,
|
| 630 |
+
j_pos // ratio: j_pos // ratio + j_len // ratio]
|
| 631 |
+
decoded = {}
|
| 632 |
+
by_shape: dict[tuple, list] = {}
|
| 633 |
+
for key, tile in tiles.items():
|
| 634 |
+
by_shape.setdefault(tuple(tile.shape), []).append(key)
|
| 635 |
+
for keys in by_shape.values():
|
| 636 |
+
for start in range(0, len(keys), max_tiles):
|
| 637 |
+
chunk = keys[start: start + max_tiles]
|
| 638 |
+
out = vae.decoder(vae.post_quant_conv(torch.cat([tiles[k] for k in chunk], dim=0)))
|
| 639 |
+
for index, key in enumerate(chunk):
|
| 640 |
+
decoded[key] = out[index: index + 1]
|
| 641 |
+
rows = [[decoded[(i, j)] for j in range(len(x_indices))] for i in range(len(y_indices))]
|
| 642 |
+
return vae._stitch_tiles(rows, y_overlaps, x_overlaps)
|
| 643 |
+
|
| 644 |
+
vae._decode_clip = decode_clip
|
| 645 |
+
|
| 646 |
+
|
| 647 |
+
# The Space's pipeline ------------------------------------------------------------------------------------------------
|
| 648 |
+
|
| 649 |
+
|
| 650 |
+
def load_slim_pipeline(path: str, adaln_calls: list, device: str = "cuda", log=print):
|
| 651 |
+
"""p-h3-image from a local snapshot, slimmed for a 48 GB slice. Components are loaded and slimmed one at a time
|
| 652 |
+
(bf16 transformer peak ~66 GB host RAM) and moved to `device`. Returns the pipeline and the upsampler's MTP head."""
|
| 653 |
+
import gc
|
| 654 |
+
|
| 655 |
+
from diffusers import ModularPipeline
|
| 656 |
+
|
| 657 |
+
start = time.time()
|
| 658 |
+
pipe = ModularPipeline.from_pretrained(path)
|
| 659 |
+
|
| 660 |
+
pipe.load_components(names=["transformer"], dtype=torch.bfloat16)
|
| 661 |
+
install_adaln_tables(pipe.transformer, [torch.tensor(call) for call in adaln_calls])
|
| 662 |
+
quantize_fp8(pipe.transformer.transformer_blocks)
|
| 663 |
+
fuse_dit_blocks(pipe.transformer)
|
| 664 |
+
pipe.transformer.to(device)
|
| 665 |
+
gc.collect()
|
| 666 |
+
log(f"[{time.time() - start:6.1f}s] transformer ready")
|
| 667 |
+
|
| 668 |
+
pipe.load_components(names=["prompt_upsampler", "prompt_upsampler_processor"], dtype=torch.bfloat16)
|
| 669 |
+
upsampler = pipe.prompt_upsampler
|
| 670 |
+
quantize_fp8(upsampler.model.language_model)
|
| 671 |
+
upsampler.lm_head = FP8Linear(upsampler.lm_head)
|
| 672 |
+
mtp = load_mtp_head(upsampler)
|
| 673 |
+
quantize_fp8(mtp)
|
| 674 |
+
install_decode_attention(upsampler.model.language_model.config, mtp.config)
|
| 675 |
+
upsampler.to(device)
|
| 676 |
+
mtp.to(device)
|
| 677 |
+
gc.collect()
|
| 678 |
+
log(f"[{time.time() - start:6.1f}s] prompt upsampler ready")
|
| 679 |
+
|
| 680 |
+
pipe.load_components(dtype=torch.bfloat16)
|
| 681 |
+
quantize_fp8(pipe.text_encoder.model.language_model)
|
| 682 |
+
quantize_fp8(pipe.vae.decoder.transformer_blocks)
|
| 683 |
+
batch_vae_tiles(pipe.vae)
|
| 684 |
+
pipe.to(device)
|
| 685 |
+
gc.collect()
|
| 686 |
+
log(f"[{time.time() - start:6.1f}s] pipeline ready")
|
| 687 |
+
return pipe, mtp
|
| 688 |
+
|
| 689 |
+
|
| 690 |
+
# Fused DiT block ----------------------------------------------------------------------------------------------------
|
| 691 |
+
#
|
| 692 |
+
# One MiniMaxH3TransformerBlock in five row kernels around the FP8 GEMMs and attention. Every producer of a GEMM input
|
| 693 |
+
# holds the whole row, so it also computes the row's FP8 scale and writes e4m3 directly (no separate quant pass):
|
| 694 |
+
# norm1 + AdaLN modulation -> FP8 | QKV GEMM | q/k RMSNorm + RoPE | attention | out GEMM
|
| 695 |
+
# gated residual + norm2 + modulation -> FP8 | FF-in GEMM | SwiGLU -> FP8 | FF-out GEMM | gated residual
|
| 696 |
+
# Math runs in fp32 and rounds once per output (eager rounds to bf16 after every op).
|
| 697 |
+
|
| 698 |
+
|
| 699 |
+
@triton.jit
|
| 700 |
+
def _rms_mod_fp8_kernel(x_ptr, w_ptr, scale_ptr, shift_ptr, idx_ptr, q_ptr, s_ptr, n_cols, stride_x, stride_tab, eps,
|
| 701 |
+
BLOCK: tl.constexpr):
|
| 702 |
+
row = tl.program_id(0).to(tl.int64)
|
| 703 |
+
cols = tl.arange(0, BLOCK)
|
| 704 |
+
mask = cols < n_cols
|
| 705 |
+
x = tl.load(x_ptr + row * stride_x + cols, mask=mask, other=0.0).to(tl.float32)
|
| 706 |
+
w = tl.load(w_ptr + cols, mask=mask, other=0.0).to(tl.float32)
|
| 707 |
+
tab = tl.load(idx_ptr + row).to(tl.int64) * stride_tab + cols
|
| 708 |
+
sc = tl.load(scale_ptr + tab, mask=mask, other=0.0).to(tl.float32)
|
| 709 |
+
sh = tl.load(shift_ptr + tab, mask=mask, other=0.0).to(tl.float32)
|
| 710 |
+
y = x * tl.math.rsqrt(tl.sum(x * x, axis=0) / n_cols + eps) * w * (1.0 + sc) + sh
|
| 711 |
+
amax = tl.maximum(tl.max(tl.where(mask, tl.abs(y), 0.0), axis=0), 1e-12)
|
| 712 |
+
tl.store(q_ptr + row * n_cols + cols, (y * (448.0 / amax)).to(tl.float8e4nv), mask=mask)
|
| 713 |
+
tl.store(s_ptr + row, amax / 448.0)
|
| 714 |
+
|
| 715 |
+
|
| 716 |
+
@triton.jit
|
| 717 |
+
def _res_gate_rms_mod_fp8_kernel(h_ptr, b_ptr, gate_ptr, w_ptr, scale_ptr, shift_ptr, idx_ptr, out_ptr, q_ptr, s_ptr,
|
| 718 |
+
n_cols, stride_h, stride_b, stride_tab, eps, BLOCK: tl.constexpr):
|
| 719 |
+
row = tl.program_id(0).to(tl.int64)
|
| 720 |
+
cols = tl.arange(0, BLOCK)
|
| 721 |
+
mask = cols < n_cols
|
| 722 |
+
tab = tl.load(idx_ptr + row).to(tl.int64) * stride_tab + cols
|
| 723 |
+
h = tl.load(h_ptr + row * stride_h + cols, mask=mask, other=0.0).to(tl.float32)
|
| 724 |
+
b = tl.load(b_ptr + row * stride_b + cols, mask=mask, other=0.0).to(tl.float32)
|
| 725 |
+
g = tl.load(gate_ptr + tab, mask=mask, other=0.0).to(tl.float32)
|
| 726 |
+
h = (h + g * b).to(out_ptr.dtype.element_ty)
|
| 727 |
+
tl.store(out_ptr + row * n_cols + cols, h, mask=mask)
|
| 728 |
+
h = h.to(tl.float32)
|
| 729 |
+
w = tl.load(w_ptr + cols, mask=mask, other=0.0).to(tl.float32)
|
| 730 |
+
sc = tl.load(scale_ptr + tab, mask=mask, other=0.0).to(tl.float32)
|
| 731 |
+
sh = tl.load(shift_ptr + tab, mask=mask, other=0.0).to(tl.float32)
|
| 732 |
+
y = h * tl.math.rsqrt(tl.sum(h * h, axis=0) / n_cols + eps) * w * (1.0 + sc) + sh
|
| 733 |
+
amax = tl.maximum(tl.max(tl.where(mask, tl.abs(y), 0.0), axis=0), 1e-12)
|
| 734 |
+
tl.store(q_ptr + row * n_cols + cols, (y * (448.0 / amax)).to(tl.float8e4nv), mask=mask)
|
| 735 |
+
tl.store(s_ptr + row, amax / 448.0)
|
| 736 |
+
|
| 737 |
+
|
| 738 |
+
@triton.jit
|
| 739 |
+
def _res_gate_kernel(h_ptr, b_ptr, gate_ptr, idx_ptr, out_ptr, n_cols, stride_h, stride_b, stride_tab,
|
| 740 |
+
BLOCK: tl.constexpr):
|
| 741 |
+
row = tl.program_id(0).to(tl.int64)
|
| 742 |
+
cols = tl.arange(0, BLOCK)
|
| 743 |
+
mask = cols < n_cols
|
| 744 |
+
tab = tl.load(idx_ptr + row).to(tl.int64) * stride_tab + cols
|
| 745 |
+
h = tl.load(h_ptr + row * stride_h + cols, mask=mask, other=0.0).to(tl.float32)
|
| 746 |
+
b = tl.load(b_ptr + row * stride_b + cols, mask=mask, other=0.0).to(tl.float32)
|
| 747 |
+
g = tl.load(gate_ptr + tab, mask=mask, other=0.0).to(tl.float32)
|
| 748 |
+
tl.store(out_ptr + row * n_cols + cols, (h + g * b).to(out_ptr.dtype.element_ty), mask=mask)
|
| 749 |
+
|
| 750 |
+
|
| 751 |
+
@triton.jit
|
| 752 |
+
def _swiglu_fp8_kernel(x_ptr, q_ptr, s_ptr, n_cols, stride_x, BLOCK: tl.constexpr):
|
| 753 |
+
row = tl.program_id(0).to(tl.int64)
|
| 754 |
+
cols = tl.arange(0, BLOCK)
|
| 755 |
+
mask = cols < n_cols
|
| 756 |
+
v = tl.load(x_ptr + row * stride_x + cols, mask=mask, other=0.0).to(tl.float32)
|
| 757 |
+
g = tl.load(x_ptr + row * stride_x + n_cols + cols, mask=mask, other=0.0).to(tl.float32)
|
| 758 |
+
y = v * g * tl.sigmoid(g)
|
| 759 |
+
amax = tl.maximum(tl.max(tl.where(mask, tl.abs(y), 0.0), axis=0), 1e-12)
|
| 760 |
+
tl.store(q_ptr + row * n_cols + cols, (y * (448.0 / amax)).to(tl.float8e4nv), mask=mask)
|
| 761 |
+
tl.store(s_ptr + row, amax / 448.0)
|
| 762 |
+
|
| 763 |
+
|
| 764 |
+
@triton.jit
|
| 765 |
+
def _qk_norm_rope_kernel(qkv_ptr, wq_ptr, wk_ptr, cos_ptr, sin_ptr, q_ptr, k_ptr, stride_qkv, stride_rope, heads,
|
| 766 |
+
eps, D: tl.constexpr, ROT: tl.constexpr):
|
| 767 |
+
row = tl.program_id(0).to(tl.int64)
|
| 768 |
+
which = tl.program_id(1) # [0, heads) query heads, [heads, 2 * heads) key heads
|
| 769 |
+
is_k = which >= heads
|
| 770 |
+
head = which - heads * is_k.to(tl.int32)
|
| 771 |
+
c = tl.arange(0, D)
|
| 772 |
+
half = ROT // 2
|
| 773 |
+
partner = tl.where(c < half, c + half, tl.where(c < ROT, c - half, c))
|
| 774 |
+
base = qkv_ptr + row * stride_qkv + (is_k.to(tl.int64) * heads + head) * D
|
| 775 |
+
x = tl.load(base + c).to(tl.float32)
|
| 776 |
+
xp = tl.load(base + partner).to(tl.float32)
|
| 777 |
+
w = tl.where(is_k, tl.load(wk_ptr + c), tl.load(wq_ptr + c)).to(tl.float32)
|
| 778 |
+
wp = tl.where(is_k, tl.load(wk_ptr + partner), tl.load(wq_ptr + partner)).to(tl.float32)
|
| 779 |
+
r = tl.math.rsqrt(tl.sum(x * x, axis=0) / D + eps)
|
| 780 |
+
xn, xpn = x * r * w, xp * r * wp
|
| 781 |
+
rot = c < ROT
|
| 782 |
+
cos = tl.load(cos_ptr + row * stride_rope + c, mask=rot, other=1.0)
|
| 783 |
+
sin = tl.load(sin_ptr + row * stride_rope + c, mask=rot, other=0.0)
|
| 784 |
+
y = (xn * cos + tl.where(c < half, -xpn, xpn) * sin).to(q_ptr.dtype.element_ty)
|
| 785 |
+
offset = (row * heads + head) * D + c
|
| 786 |
+
tl.store(q_ptr + offset, y, mask=(c < D) & (which < heads))
|
| 787 |
+
tl.store(k_ptr + offset, y, mask=(c < D) & (which >= heads))
|
| 788 |
+
|
| 789 |
+
|
| 790 |
+
def _row_launch(kernel, rows, cols, *args, **kwargs):
|
| 791 |
+
block = triton.next_power_of_2(cols)
|
| 792 |
+
kernel[(rows,)](*args, BLOCK=block, num_warps=16 if block >= 16384 else 8, **kwargs)
|
| 793 |
+
|
| 794 |
+
|
| 795 |
+
def _fp8_gemm(q, s, linear: FP8Linear):
|
| 796 |
+
return torch._scaled_mm(q, linear.weight.t(), scale_a=s, scale_b=linear.weight_scale, bias=linear.bias,
|
| 797 |
+
out_dtype=linear.out_dtype, use_fast_accum=linear.fast_accum)
|
| 798 |
+
|
| 799 |
+
|
| 800 |
+
def _fused_block_forward(block, hidden_states, temb, adaln_indices, rotary_emb, attention_mask=None):
|
| 801 |
+
from diffusers.models.attention_dispatch import dispatch_attention_fn
|
| 802 |
+
|
| 803 |
+
shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = block.adaln_proj(temb)
|
| 804 |
+
batch, rows, cols = hidden_states.shape
|
| 805 |
+
h = hidden_states.reshape(rows, cols)
|
| 806 |
+
idx, stride_tab, device = adaln_indices, shift_msa.stride(0), h.device
|
| 807 |
+
attn = block.attn
|
| 808 |
+
heads, dim = attn.heads, attn.head_dim
|
| 809 |
+
q8 = torch.empty((rows, cols), device=device, dtype=FP8)
|
| 810 |
+
s = torch.empty((rows, 1), device=device, dtype=torch.float32)
|
| 811 |
+
_row_launch(_rms_mod_fp8_kernel, rows, cols, h, block.norm1.weight, scale_msa, shift_msa, idx, q8, s, cols,
|
| 812 |
+
h.stride(0), stride_tab, block.norm1.eps)
|
| 813 |
+
qkv = _fp8_gemm(q8, s, block._fused_qkv)
|
| 814 |
+
q = torch.empty((rows, heads, dim), device=device, dtype=h.dtype)
|
| 815 |
+
k = torch.empty_like(q)
|
| 816 |
+
cos, sin = rotary_emb
|
| 817 |
+
_qk_norm_rope_kernel[(rows, 2 * heads)](qkv, attn.norm_q.weight, attn.norm_k.weight, cos, sin, q, k, qkv.stride(0),
|
| 818 |
+
cos.stride(0), heads, attn.norm_q.eps, D=dim, ROT=cos.shape[-1],
|
| 819 |
+
num_warps=2)
|
| 820 |
+
v = qkv[:, 2 * heads * dim:].view(rows, heads, dim)
|
| 821 |
+
o = dispatch_attention_fn(q[None], k[None], v[None], attn_mask=None, dropout_p=0.0, is_causal=False,
|
| 822 |
+
backend=attn.processor._attention_backend)
|
| 823 |
+
attn_out = attn.to_out[0](o.reshape(rows, heads * dim))
|
| 824 |
+
h2 = torch.empty_like(h)
|
| 825 |
+
_row_launch(_res_gate_rms_mod_fp8_kernel, rows, cols, h, attn_out, gate_msa, block.norm2.weight, scale_mlp,
|
| 826 |
+
shift_mlp, idx, h2, q8, s, cols, h.stride(0), attn_out.stride(0), stride_tab, block.norm2.eps)
|
| 827 |
+
proj = _fp8_gemm(q8, s, block.ff.net[0].proj)
|
| 828 |
+
inner = proj.shape[1] // 2
|
| 829 |
+
f8 = torch.empty((rows, inner), device=device, dtype=FP8)
|
| 830 |
+
_row_launch(_swiglu_fp8_kernel, rows, inner, proj, f8, s, inner, proj.stride(0))
|
| 831 |
+
ff_out = _fp8_gemm(f8, s, block.ff.net[2])
|
| 832 |
+
out = torch.empty_like(h)
|
| 833 |
+
_row_launch(_res_gate_kernel, rows, cols, h2, ff_out, gate_mlp, idx, out, cols, h2.stride(0), ff_out.stride(0),
|
| 834 |
+
stride_tab)
|
| 835 |
+
return out.view(batch, rows, cols)
|
| 836 |
+
|
| 837 |
+
|
| 838 |
+
@torch.no_grad()
|
| 839 |
+
def fuse_dit_blocks(transformer) -> None:
|
| 840 |
+
"""Fused forwards for every transformer block. Needs `install_adaln_tables` and `quantize_fp8` first (and q, k, v
|
| 841 |
+
as separate FP8Linear, which are concatenated into one QKV GEMM here)."""
|
| 842 |
+
import types
|
| 843 |
+
|
| 844 |
+
for block in transformer.transformer_blocks:
|
| 845 |
+
attn = block.attn
|
| 846 |
+
parts = [attn.to_q, attn.to_k, attn.to_v]
|
| 847 |
+
fused = FP8Linear.__new__(FP8Linear)
|
| 848 |
+
nn.Module.__init__(fused)
|
| 849 |
+
fused.register_buffer("weight", torch.cat([p.weight for p in parts]))
|
| 850 |
+
fused.register_buffer("weight_scale", torch.cat([p.weight_scale for p in parts], dim=1))
|
| 851 |
+
fused.bias, fused.out_dtype, fused.fast_accum = None, parts[0].out_dtype, parts[0].fast_accum
|
| 852 |
+
fused.in_features, fused.out_features = parts[0].in_features, sum(p.out_features for p in parts)
|
| 853 |
+
block._fused_qkv = fused
|
| 854 |
+
del attn.to_q, attn.to_k, attn.to_v
|
| 855 |
+
block.forward = types.MethodType(_fused_block_forward, block)
|
| 856 |
+
_free_cache()
|
| 857 |
+
|
requirements.txt
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
torch==2.13.0
|
| 2 |
+
torchvision==0.28.0
|
| 3 |
+
transformers==5.20.2
|
| 4 |
+
kernels==0.17.2
|
| 5 |
+
einops
|
| 6 |
+
accelerate
|
| 7 |
+
safetensors
|
| 8 |
+
huggingface_hub
|
| 9 |
+
pillow
|
| 10 |
+
numpy
|
| 11 |
+
diffusers @ git+https://github.com/huggingface/diffusers@c55ec27943462eeeb65c177d6ff7d8272d21d7a7
|