nifleisch commited on
Commit
9d61ded
·
verified ·
1 Parent(s): 5f954be

P-H3-Image demo

Browse files
Files changed (6) hide show
  1. README.md +10 -8
  2. adaln_calls.json +1 -0
  3. app.py +246 -0
  4. examples.py +84 -0
  5. p_h3_image_fast.py +857 -0
  6. requirements.txt +11 -0
README.md CHANGED
@@ -1,13 +1,15 @@
1
  ---
2
- title: P H3 Image
3
- emoji: 🐠
4
- colorFrom: indigo
5
- colorTo: yellow
6
  sdk: gradio
7
- sdk_version: 6.31.0
8
- python_version: '3.13'
9
  app_file: app.py
10
  pinned: false
 
 
 
 
11
  ---
12
-
13
- Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
1
  ---
2
+ title: P-H3-Image
3
+ emoji: 🖼️
4
+ colorFrom: gray
5
+ colorTo: gray
6
  sdk: gradio
7
+ sdk_version: 6.28.0
8
+ python_version: "3.12"
9
  app_file: app.py
10
  pinned: false
11
+ startup_duration_timeout: 1h
12
+ short_description: Text-to-image and image editing, based on MiniMax-H3
13
+ models:
14
+ - PrunaAI/P-H3-Image
15
  ---
 
 
adaln_calls.json ADDED
@@ -0,0 +1 @@
 
 
1
+ [[0.0], [0.0, 0.9990000128746033], [0.015246987342834473], [0.015246987342834473, 0.9990000128746033], [0.03209501504898071], [0.03209501504898071, 0.9990000128746033], [0.04831200838088989], [0.04831200838088989, 0.9990000128746033], [0.07462799549102783], [0.07462799549102783, 0.9990000128746033], [0.11807698011398315], [0.11807698011398315, 0.9990000128746033], [0.18991702795028687], [0.18991702795028687, 0.9990000128746033], [0.4712820053100586], [0.4712820053100586, 0.9990000128746033]]
app.py ADDED
@@ -0,0 +1,246 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """P-H3-Image on ZeroGPU (`xlarge`: a full RTX PRO 6000 Blackwell, 96 GB).
2
+
3
+ The pipeline is loaded once at startup, one component at a time (the host has ~97 GB), and slimmed to fit the slice
4
+ with p_h3_image_fast.py (kernels only, no torch.compile): AdaLN tables instead of the transformer's AdaLN projections,
5
+ FP8 linears everywhere, fused Triton DiT blocks, batched VAE tiles, and for the prompt upsampler a weight-only FP8 GEMV,
6
+ decode attention and speculative decoding with its MTP head in one CUDA graph per step.
7
+ """
8
+
9
+ import json
10
+ import os
11
+ import random
12
+ import tempfile
13
+ import time
14
+
15
+ os.environ.setdefault("DIFFUSERS_TRUST_REMOTE_KERNELS", "true")
16
+ os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
17
+
18
+ import gradio as gr
19
+ import spaces
20
+ import torch
21
+ from huggingface_hub import snapshot_download
22
+ from PIL import Image, ImageOps
23
+ from torchvision.io import encode_jpeg
24
+
25
+ import p_h3_image_fast as fast
26
+ from diffusers import ModularPipeline
27
+ from examples import EXAMPLES
28
+
29
+ REPO, REVISION = "PrunaAI/P-H3-Image", "ee4a685de1fbd4337a50bcf9b3c407e4f8463d49"
30
+ # Canvases (width, height), multiples of 32: 4 and 8 MP as tested, 16 MP from the same ratios.
31
+ RATIOS = {"1:1": 1.0, "4:3": 4 / 3, "3:4": 3 / 4, "3:2": 3 / 2, "2:3": 2 / 3, "16:9": 16 / 9, "9:16": 9 / 16}
32
+ T2I_RESOLUTIONS, EDIT_RESOLUTIONS = ["4 MP", "8 MP", "16 MP"], ["4 MP", "8 MP"]
33
+
34
+
35
+ def _canvas(ratio: float, megapixels: int) -> tuple[int, int]:
36
+ pixels = megapixels * 1024**2
37
+ return round((pixels * ratio) ** 0.5 / 32) * 32, round((pixels / ratio) ** 0.5 / 32) * 32
38
+
39
+
40
+ SIZES = {
41
+ "4 MP": {"1:1": (2048, 2048), "4:3": (2368, 1760), "3:4": (1760, 2368), "3:2": (2496, 1664), "2:3": (1664, 2496),
42
+ "16:9": (2720, 1536), "9:16": (1536, 2720)},
43
+ "8 MP": {"1:1": (2912, 2912), "4:3": (3360, 2496), "3:4": (2496, 3360), "3:2": (3552, 2368), "2:3": (2368, 3552),
44
+ "16:9": (3872, 2176), "9:16": (2176, 3872)},
45
+ "16 MP": {name: _canvas(ratio, 16) for name, ratio in RATIOS.items()},
46
+ }
47
+ MAX_IMAGES = 8
48
+ # All references together stay within 8 MP: one image keeps up to 8 MP, eight get 1 MP each.
49
+ REFERENCE_BUDGET = 8 * 1024**2
50
+
51
+
52
+ def load_pipeline():
53
+ path = snapshot_download(REPO, revision=REVISION, max_workers=16)
54
+ with open(os.path.join(os.path.dirname(__file__), "adaln_calls.json")) as handle:
55
+ calls = json.load(handle)
56
+ pipe, mtp = fast.load_slim_pipeline(path, calls, log=lambda message: print(message, flush=True))
57
+ # The same components without the prompt upsampler step, for requests that skip upsampling.
58
+ blocks = ModularPipeline.from_pretrained(path).blocks
59
+ blocks.sub_blocks.pop("prompt_upsampler")
60
+ plain = blocks.init_pipeline(path)
61
+ plain.update_components(**{name: getattr(pipe, name) for name in plain.component_names
62
+ if getattr(pipe, name, None) is not None})
63
+ return pipe, plain, mtp
64
+
65
+
66
+ PIPE, PLAIN, MTP = load_pipeline()
67
+ UPSAMPLER_GENERATE = PIPE.prompt_upsampler.generate
68
+ STATE = {"kernels": False, "decoder": None, "owner": None}
69
+
70
+
71
+ def _prepare_gpu():
72
+ """Per GPU call. Once per worker process: the upsampler's hub kernels, and the speculative decoder with its CUDA
73
+ graph (captured on the first request, then replayed; rebuilt if the process or device changes)."""
74
+ upsampler = PIPE.prompt_upsampler
75
+ if not STATE["kernels"]:
76
+ from kernels import Mode
77
+
78
+ upsampler.set_use_kernels(True, mode=Mode.INFERENCE)
79
+ STATE["kernels"] = True
80
+ # The graph bakes in weight addresses: rebuild if ZeroGPU placed the weights elsewhere since the capture.
81
+ weights = (upsampler.lm_head.weight, MTP.fc.weight, upsampler.model.language_model.embed_tokens.weight,
82
+ upsampler.model.language_model.layers[0].mlp.down_proj.weight)
83
+ owner = (os.getpid(), torch.cuda.current_device(), tuple(w.data_ptr() for w in weights))
84
+ if STATE["decoder"] is None or STATE["owner"] != owner:
85
+ upsampler.generate = UPSAMPLER_GENERATE
86
+ STATE["decoder"], STATE["owner"] = fast.MTPGreedyGenerate(upsampler, MTP), owner
87
+ decoder = STATE["decoder"]
88
+ timing = {}
89
+
90
+ def generate(*args, **kwargs):
91
+ torch.cuda.synchronize()
92
+ begin = time.perf_counter()
93
+ out = decoder.generate(*args, **kwargs)
94
+ torch.cuda.synchronize()
95
+ timing["upsampler"] = time.perf_counter() - begin
96
+ timing["tokens"] = decoder.stats.get("tokens", 0)
97
+ return out
98
+
99
+ upsampler.generate = generate
100
+ return timing
101
+
102
+
103
+ def _duration(mode, prompt, images, aspect, resolution, upsample, seed):
104
+ # Measured on a full card: ~6 / 12 / 25 s for 4 / 8 / 16 MP text-to-image, edits a bit more per image. Shorter
105
+ # requests queue faster on ZeroGPU.
106
+ base = {"4 MP": 25, "8 MP": 35, "16 MP": 60}[resolution]
107
+ return base + (10 + 5 * len(images) if mode == "edit" else 0)
108
+
109
+
110
+ @spaces.GPU(size="xlarge", duration=_duration)
111
+ def _render(mode, prompt, images, aspect, resolution, upsample, seed):
112
+ timing = _prepare_gpu()
113
+ torch.cuda.synchronize()
114
+ start = time.perf_counter()
115
+ kwargs = {"prompt": prompt, "generator": torch.Generator().manual_seed(seed)}
116
+ if mode == "edit":
117
+ # The first image's aspect ratio at the chosen size.
118
+ width, height = images[0].size
119
+ kwargs["image"] = images
120
+ kwargs["width"], kwargs["height"] = _canvas(width / height, int(resolution.split()[0]))
121
+ else:
122
+ kwargs["width"], kwargs["height"] = SIZES[resolution][aspect]
123
+ result = (PIPE if upsample else PLAIN)(**kwargs, output_type="pt", output=["images", "prompt"])
124
+ path, size = _to_jpeg(result["images"])
125
+ total = time.perf_counter() - start
126
+ return path, size, result["prompt"], {"total": total, **timing}
127
+
128
+
129
+ def _to_jpeg(images: torch.Tensor) -> tuple[str, tuple[int, int]]:
130
+ """The decoded image as a JPEG file: uint8 and nvjpeg on the GPU, so only the encoded bytes leave the worker."""
131
+ pixels = (images[0].clamp(0, 1) * 255).round().to(torch.uint8).contiguous()
132
+ try:
133
+ data = encode_jpeg(pixels, quality=93)
134
+ except Exception: # noqa: BLE001 - no nvjpeg: encode on the CPU
135
+ data = encode_jpeg(pixels.cpu(), quality=93)
136
+ with tempfile.NamedTemporaryFile(suffix=".jpg", delete=False) as handle:
137
+ handle.write(data.cpu().numpy().tobytes())
138
+ return handle.name, (pixels.shape[2], pixels.shape[1])
139
+
140
+
141
+ @spaces.GPU(size="xlarge", duration=300)
142
+ def _warmup():
143
+ """Triton compiles and autotunes, and the hub kernels load, before the first visitor."""
144
+ image = Image.new("RGB", (1024, 1024), (120, 140, 160))
145
+ timing = _prepare_gpu()
146
+ PIPE(prompt="a lighthouse on a cliff at dusk", width=1024, height=1024, generator=torch.Generator().manual_seed(0),
147
+ output="images")
148
+ PIPE(prompt="make it night", image=[image], width=1024, height=1024, generator=torch.Generator().manual_seed(0),
149
+ output="images")
150
+ return timing
151
+
152
+
153
+ try:
154
+ _warmup()
155
+ except Exception as error: # noqa: BLE001 - the demo still works, the first request just pays the warmup
156
+ print(f"warmup failed: {error!r}", flush=True)
157
+
158
+
159
+ def _info(timing, size, seed):
160
+ upsample, tokens = timing.get("upsampler", 0.0), timing.get("tokens", 0)
161
+ rate = f", {tokens} tokens at {tokens / upsample:.0f} tokens/s" if upsample and tokens else ""
162
+ head = f"{size[0]}×{size[1]}, seed {seed} · {timing['total']:.1f} s on the GPU"
163
+ if not upsample:
164
+ return f"{head} (no prompt upsampling)"
165
+ return f"{head} ({upsample:.1f} s prompt upsampling{rate}, {timing['total'] - upsample:.1f} s rendering)"
166
+
167
+
168
+ def text_to_image(prompt, aspect, resolution, upsample):
169
+ if not prompt or not prompt.strip():
170
+ raise gr.Error("Write a prompt first.")
171
+ seed = random.randint(0, 2**31 - 1)
172
+ try:
173
+ path, size, upsampled, timing = _render("t2i", prompt.strip(), [], aspect, resolution, upsample, seed)
174
+ except RuntimeError as error:
175
+ raise gr.Error(str(error).split("\n")[0])
176
+ return path, upsampled, _info(timing, size, seed)
177
+
178
+
179
+ def _fit(image: Image.Image, pixels: float) -> Image.Image:
180
+ """Downscale to at most `pixels` (the pipeline keeps references at their own size, from 1 MP up)."""
181
+ scale = (pixels / (image.width * image.height)) ** 0.5
182
+ if scale >= 1:
183
+ return image
184
+ return image.resize((max(1, round(image.width * scale)), max(1, round(image.height * scale))), Image.LANCZOS)
185
+
186
+
187
+ def edit(gallery, prompt, resolution, upsample):
188
+ images = []
189
+ for item in gallery or []:
190
+ item = item[0] if isinstance(item, (tuple, list)) else item
191
+ item = Image.open(item) if isinstance(item, str) else item
192
+ images.append(ImageOps.exif_transpose(item).convert("RGB"))
193
+ if not images:
194
+ raise gr.Error("Add at least one image; the first one is edited.")
195
+ if len(images) > MAX_IMAGES:
196
+ raise gr.Error(f"At most {MAX_IMAGES} images.")
197
+ if not prompt or not prompt.strip():
198
+ raise gr.Error("Describe the edit first.")
199
+ images = [_fit(image, REFERENCE_BUDGET / len(images)) for image in images]
200
+ seed = random.randint(0, 2**31 - 1)
201
+ try:
202
+ path, size, upsampled, timing = _render("edit", prompt.strip(), images, None, resolution, upsample, seed)
203
+ except (RuntimeError, ValueError) as error:
204
+ raise gr.Error(str(error).split("\n")[0])
205
+ return path, upsampled, _info(timing, size, seed)
206
+
207
+
208
+ with gr.Blocks(title="P-H3-Image") as demo:
209
+ gr.Markdown("# P-H3-Image\nPruna's image model for text-to-image and image editing, based on MiniMax-H3. "
210
+ "By default every request goes through the prompt upsampler first. "
211
+ "[Model card](https://huggingface.co/PrunaAI/P-H3-Image)")
212
+ with gr.Tab("Text to image"):
213
+ with gr.Row():
214
+ with gr.Column():
215
+ t_prompt = gr.Textbox(label="Prompt", lines=4)
216
+ with gr.Row():
217
+ t_aspect = gr.Dropdown(list(RATIOS), value="4:3", label="Aspect ratio")
218
+ t_resolution = gr.Radio(T2I_RESOLUTIONS, value="8 MP", label="Resolution")
219
+ t_upsample = gr.Checkbox(value=True, label="Prompt upsampling")
220
+ t_button = gr.Button("Generate", variant="primary")
221
+ with gr.Column():
222
+ t_image = gr.Image(label="Result", type="filepath")
223
+ t_info = gr.Markdown()
224
+ with gr.Accordion("Upsampled prompt", open=False):
225
+ t_upsampled = gr.Textbox(show_label=False, lines=10)
226
+ gr.Examples([[prompt, aspect] for _, aspect, prompt in EXAMPLES], inputs=[t_prompt, t_aspect],
227
+ example_labels=[label for label, _, _ in EXAMPLES])
228
+ t_button.click(text_to_image, [t_prompt, t_aspect, t_resolution, t_upsample], [t_image, t_upsampled, t_info])
229
+ with gr.Tab("Edit"):
230
+ with gr.Row():
231
+ with gr.Column():
232
+ e_images = gr.Gallery(label=f"Images (the first one is edited, up to {MAX_IMAGES})", type="pil",
233
+ columns=4, height=260, interactive=True)
234
+ e_prompt = gr.Textbox(label="Edit", lines=3, placeholder="e.g. put the fox from <Picture 2> on the bench")
235
+ e_resolution = gr.Radio(EDIT_RESOLUTIONS, value="8 MP", label="Resolution",
236
+ info="The first image's aspect ratio at this size.")
237
+ e_upsample = gr.Checkbox(value=True, label="Prompt upsampling")
238
+ e_button = gr.Button("Edit", variant="primary")
239
+ with gr.Column():
240
+ e_image = gr.Image(label="Result", type="filepath")
241
+ e_info = gr.Markdown()
242
+ with gr.Accordion("Upsampled prompt", open=False):
243
+ e_upsampled = gr.Textbox(show_label=False, lines=10)
244
+ e_button.click(edit, [e_images, e_prompt, e_resolution, e_upsample], [e_image, e_upsampled, e_info])
245
+
246
+ demo.queue(max_size=20).launch()
examples.py ADDED
@@ -0,0 +1,84 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Text-to-image examples: (label, aspect ratio, prompt)."""
2
+
3
+ EXAMPLES = [
4
+ ("Negotiation in orbit", "16:9",
5
+ "Medium wide shot from a low angle across a long table, on a 35mm lens, with the window and far wall behind the "
6
+ "figures. Five people sit in tense negotiation, leaning toward each other. Two humans in worn grey wool suits "
7
+ "face three alien delegates. The first alien is tall and thin-limbed, with grey skin, long folded fingers and a "
8
+ "robe of coarse woven fiber. The second is squat and heavy, its back plated with pale ridged shell, gripping a "
9
+ "stylus in three thick fingers. The third has a dark mantle of fine filaments and wears a harness of dented brass "
10
+ "clips. A cracked ceramic jug, scuffed data slates and curled paper printouts sit on the table. Earth fills the "
11
+ "large curved window, a band of blue cloud over dark ocean. Ribbed steel girders frame the foreground. Cold "
12
+ "sunlight rakes across the table while one amber lamp hangs overhead, casting soft shadows under the chins. Fine "
13
+ "dust drifts in the light. Palette of slate, bone, rust and deep teal, natural film color, fine grain, shadows "
14
+ "deep with detail kept."),
15
+ ("Astronaut in a flooded jungle", "16:9",
16
+ "A frame from a feature film: a wide establishing shot from a low angle, with a small astronaut standing "
17
+ "waist-deep in a flooded jungle clearing, captured on a large-format digital cinema camera with a 35mm lens.\n\n"
18
+ "The astronaut wears a worn white spacesuit with grey mud on the knees, scuffed gloves, and a cracked visor that "
19
+ "reflects the canopy. One hand grips a rusted machete while the other rests on a broken comms pack. The helmet "
20
+ "is tilted toward a sound in the trees, caught mid-step as if listening for danger.\n\n"
21
+ "Behind the figure, tangled roots and fallen trunks rise out of black water. A pale strangler fig wraps around a "
22
+ "torn fragment of a landing hull. Farther back, ferns and mist-shrouded trunks fade into layers of blue-grey "
23
+ "haze.\n\n"
24
+ "The light is a low, overcast sky above the canopy, a cold top light with a faint blue-white bounce off the "
25
+ "water. A single battery lantern clipped to the pack casts a small warm pool across the suit.\n\n"
26
+ "Drifting mist hangs between the trunks, and fine rain dimples the water surface. The palette is restrained: "
27
+ "slate blue, steel grey, muted sage green and pale ivory, with film-like color and fine grain."),
28
+ ("Knight in a frozen forest", "3:2",
29
+ "A photograph by an experienced landscape photographer, shot at eye level from a low position with a 35mm lens "
30
+ "at f/2.8, the knight placed in the middle distance on a narrow forest path.\n\n"
31
+ "The knight is a weathered man in his forties, his beard rimmed with frost and his brow tight as he scans the "
32
+ "trees. His dented steel plate is dulled with grey scratches, a wool cloak hangs heavy and damp at his "
33
+ "shoulders, and his right hand rests loosely on a sword hilt wrapped in worn leather. His breath drifts visibly "
34
+ "in the cold air as he walks mid-stride, one boot sinking into the snow.\n\n"
35
+ "Behind him, birch and pine trunks are coated in white frost, and fallen branches lie half-buried under drifts. "
36
+ "Moonlight comes from high upper left, pale and cool, lifting the frost on the bark and the rim of his pauldron, "
37
+ "while the forest floor falls into deep blue shadow where texture remains visible. The palette stays "
38
+ "restrained: slate blue, grey-white snow, muted brown leather and tarnished steel, with neutral white balance "
39
+ "and fine film grain."),
40
+ ("Anime sword duel", "16:9",
41
+ "A frame from a Japanese anime television series, hand-drawn cel animation over a painted background, "
42
+ "widescreen composition. On a rocky mountaintop in a thunderstorm, a swordsman in a torn dark coat swings a "
43
+ "massive broadsword that glows pale blue, its blade mid-arc and his teeth clenched. Opposite him, a slender "
44
+ "fighter in a white scarf blurs into motion, leaving streaks of afterimage and a ring of cracked stone. "
45
+ "Lightning forks across the sky behind them, rain slants through the frame, and bright white highlights gleam "
46
+ "on wet rock."),
47
+ ("1970s sci-fi comic", "3:2",
48
+ "A gouache and ink illustration in the style of a 1970s European science-fiction comic, filling the whole "
49
+ "picture edge to edge. A lone space traveler in a bulky white suit stands on a rocky ledge, helmet visor "
50
+ "catching the faint light, gazing up at towering mushroom trees with broad caps and thick pale stalks. A dim "
51
+ "violet sky stretches above, with a few pale spores drifting through the air. Flat violet and ochre color with "
52
+ "visible brush texture and inked contour lines, shown in a wide landscape view from a low angle."),
53
+ ("Linen lookbook", "2:3",
54
+ "A lookbook photograph by an experienced fashion photographer: a woman in her late twenties stands full length "
55
+ "in a pale sandy-linen dress, a loose sleeveless A-line with a square neckline, a gathered waist and a hem that "
56
+ "falls just below the calf. The natural linen creases softly at the hips and lifts slightly as she shifts her "
57
+ "weight onto one foot in a light breeze. Her chestnut hair is loosely tied back, she wears simple leather "
58
+ "sandals and small gold hoops, and her expression is calm as she looks just past the camera. The location is a "
59
+ "whitewashed plaster wall beside a weathered wooden door, with the shadow of a palm frond falling across the "
60
+ "plaster. Late-afternoon sunlight from the left gives warm directional light with visible detail in the "
61
+ "shadows. Shot at 85mm from a few steps back at standing eye height, shallow depth of field, true white "
62
+ "balance, fine grain, restrained palette of cream, sand and sage. No text, no other objects."),
63
+ ("Risograph cyclist", "3:4",
64
+ "A two-color risograph print on cream paper, the artwork running edge to edge with no border or margin. A "
65
+ "cyclist in a hooded jacket and helmet leans into a turn on a bicycle down a wide city avenue, seen from a low "
66
+ "side angle. Behind them rise tall apartment blocks with rows of windows, a tram line curving into the "
67
+ "distance, and a few parked cars. Only the cyclist, the buildings and the road are shown. The image uses "
68
+ "fluorescent pink and deep blue inks, which overprint to a dark purple where they overlap. Visible halftone "
69
+ "dots, grainy ink coverage, and slight misregistration between the two passes run across the whole sheet."),
70
+ ("Byzantine mosaic", "3:4",
71
+ "A Byzantine gold-ground mosaic panel, small glass and stone tesserae set in lime mortar, with the gold tiles "
72
+ "catching light unevenly. A lighthouse keeper in a dark blue tunic and short cloak stands at the base of a tall "
73
+ "white tower, one hand raised to shield his eyes as he looks out to sea. Behind him a lantern glows with a "
74
+ "red-gold flame, and the water below is built in bands of teal and deep blue tesserae, with a small ship on the "
75
+ "horizon. The gold ground fills the background, the figures are outlined in dark grey-brown tesserae, and the "
76
+ "grout lines are slightly uneven. The mosaic fills the whole image edge to edge, with no frame or border."),
77
+ ("Kirchner street at night", "3:4",
78
+ "A German Expressionist oil painting on canvas in the manner of Ernst Ludwig Kirchner's Berlin street scenes. A "
79
+ "narrow city street at night tilts sharply toward the viewer, its buildings leaning into angular, flattened "
80
+ "planes. Gaunt figures in dark coats hurry past a gas lamp whose light is a thick yellow smear dragged across "
81
+ "the wet paving. Heavy impasto brushstrokes and visible palette-knife edges, with raw cadmium yellow, cobalt "
82
+ "blue and vermilion patches in the lit windows. The paint covers the canvas edge to edge, with no frame or "
83
+ "border."),
84
+ ]
p_h3_image_fast.py ADDED
@@ -0,0 +1,857 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Kernel-level speedups for p-h3-image (the diffusers fork) on an RTX PRO 6000 Blackwell (sm_120). No torch.compile.
2
+
3
+ * `install_adaln_tables`: every block's AdaLN projection (13B of the transformer's 33B parameters) depends only on
4
+ the timestep, and the pipeline only ever uses the timesteps of its fixed sigma grid. Their outputs are computed once
5
+ per transformer call signature and the projection weights are dropped. Bitwise identical to the projections.
6
+ * `fuse_qkv`: one GEMM for q, k and v.
7
+ * `quantize_fp8`: W8A8 linears, per-output-channel weight scales and per-row dynamic activation scales, through
8
+ torch._scaled_mm (cuBLASLt FP8). Activations are quantized by one Triton kernel per GEMM input.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import inspect
14
+ import time
15
+
16
+ import torch
17
+ import triton
18
+ import triton.language as tl
19
+ from torch import nn
20
+
21
+ FP8 = torch.float8_e4m3fn
22
+ FP8_MAX = 448.0
23
+
24
+
25
+ def _free_cache() -> None:
26
+ try:
27
+ if torch.cuda.is_available() and torch.cuda.is_initialized():
28
+ torch.cuda.empty_cache()
29
+ except Exception: # noqa: BLE001 - ZeroGPU emulates CUDA outside GPU calls
30
+ pass
31
+
32
+
33
+ # AdaLN tables -------------------------------------------------------------------------------------------------------
34
+
35
+
36
+ class AdaLNTable(nn.Module):
37
+ """Replaces `MiniMaxH3AdaLayerNormModulation`: returns its precomputed output for the current call's timesteps."""
38
+
39
+ def __init__(self, tables: list[torch.Tensor], hidden_size: int, lookup: dict):
40
+ super().__init__()
41
+ for index, table in enumerate(tables):
42
+ self.register_buffer(f"table_{index}", table, persistent=False)
43
+ self.hidden_size = hidden_size
44
+ self.lookup = lookup
45
+
46
+ def forward(self, temb: torch.Tensor) -> tuple[torch.Tensor, ...]:
47
+ table = getattr(self, f"table_{self.lookup['index']}")
48
+ return table.view(-1, 6 * self.hidden_size).chunk(6, dim=-1)
49
+
50
+
51
+ @torch.no_grad()
52
+ def install_adaln_tables(transformer, calls: list[torch.Tensor]) -> int:
53
+ """Precompute every block's AdaLN output for each distinct `timestep` argument in `calls` (as the pipeline passes
54
+ them, one tensor per transformer call) and drop the projection weights. Each table is computed with the call's own
55
+ shapes, exactly as the forward would, so the outputs are bitwise identical."""
56
+ device = next(transformer.parameters()).device
57
+ keys: dict[tuple, int] = {}
58
+ tembs = []
59
+ for timestep in calls:
60
+ key = tuple(timestep.tolist())
61
+ if key not in keys:
62
+ keys[key] = len(keys)
63
+ temb = transformer.time_proj(timestep.to(device))
64
+ tembs.append(transformer.time_embedder(temb.to(transformer.time_embedder.linear_1.weight.dtype)))
65
+ lookup = {"keys": keys, "index": None}
66
+ freed = 0
67
+ for block in transformer.transformer_blocks:
68
+ proj = block.adaln_proj
69
+ dtype = proj.linear.weight.dtype
70
+ tables = [proj.linear(nn.functional.silu(temb).to(dtype)).contiguous() for temb in tembs]
71
+ freed += sum(p.numel() for p in proj.parameters())
72
+ block.adaln_proj = AdaLNTable(tables, proj.hidden_size, lookup)
73
+ del proj
74
+
75
+ def set_index(module, args, kwargs):
76
+ key = tuple(kwargs["timestep"].tolist())
77
+ if key not in keys:
78
+ raise ValueError(f"No AdaLN table for timesteps {key}; known: {list(keys)}")
79
+ lookup["index"] = keys[key]
80
+
81
+ transformer.register_forward_pre_hook(set_index, with_kwargs=True)
82
+ _free_cache()
83
+ return freed
84
+
85
+
86
+ # QKV fusion ---------------------------------------------------------------------------------------------------------
87
+
88
+
89
+ @torch.no_grad()
90
+ def fuse_qkv(transformer) -> None:
91
+ for block in transformer.transformer_blocks:
92
+ attn = block.attn
93
+ weight = torch.cat([attn.to_q.weight, attn.to_k.weight, attn.to_v.weight])
94
+ attn.to_qkv = nn.Linear(weight.shape[1], weight.shape[0], bias=False, device=weight.device, dtype=weight.dtype)
95
+ attn.to_qkv.weight.copy_(weight)
96
+ del attn.to_q, attn.to_k, attn.to_v
97
+ attn.fused_projections = True
98
+ _free_cache()
99
+
100
+
101
+ # FP8 W8A8 -----------------------------------------------------------------------------------------------------------
102
+
103
+
104
+ @triton.jit
105
+ def _rowwise_fp8_kernel(x_ptr, q_ptr, s_ptr, K, stride_x, stride_q, BLOCK: tl.constexpr):
106
+ row = tl.program_id(0)
107
+ offs = tl.arange(0, BLOCK)
108
+ mask = offs < K
109
+ x = tl.load(x_ptr + row * stride_x + offs, mask=mask, other=0.0).to(tl.float32)
110
+ amax = tl.maximum(tl.max(tl.abs(x), axis=0), 1e-12)
111
+ tl.store(q_ptr + row * stride_q + offs, (x * (448.0 / amax)).to(tl.float8e4nv), mask=mask)
112
+ tl.store(s_ptr + row, amax / 448.0)
113
+
114
+
115
+ def rowwise_fp8(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
116
+ """(M, K) bf16 -> (M, K) e4m3 and (M, 1) float32 scales with x ~= q * scale."""
117
+ M, K = x.shape
118
+ q = torch.empty((M, K), device=x.device, dtype=FP8)
119
+ scale = torch.empty((M, 1), device=x.device, dtype=torch.float32)
120
+ block = triton.next_power_of_2(K)
121
+ _rowwise_fp8_kernel[(M,)](x, q, scale, K, x.stride(0), q.stride(0), BLOCK=block, num_warps=8 if block <= 8192 else 16)
122
+ return q, scale
123
+
124
+
125
+ _W8A16_CONFIGS = [triton.Config({"BLOCK_N": n, "BLOCK_K": k}, num_warps=w, num_stages=4)
126
+ for n in (16, 32, 64) for k in (128, 256) for w in (4, 8)]
127
+
128
+
129
+ # Tuned configs persist in Triton's cache dir, so a new process (ZeroGPU workers) skips the autotune.
130
+ _AUTOTUNE_CACHE = {"cache_results": True} if "cache_results" in inspect.signature(triton.autotune).parameters else {}
131
+
132
+
133
+ @triton.autotune(configs=_W8A16_CONFIGS, key=["N", "K"], **_AUTOTUNE_CACHE)
134
+ @triton.jit
135
+ def _w8a16_kernel(x_ptr, w_ptr, s_ptr, b_ptr, y_ptr, M, N, K, stride_x, stride_w, stride_y,
136
+ HAS_BIAS: tl.constexpr, BLOCK_N: tl.constexpr, BLOCK_K: tl.constexpr):
137
+ offs_n = tl.program_id(0) * BLOCK_N + tl.arange(0, BLOCK_N)
138
+ offs_m = tl.arange(0, 16)
139
+ acc = tl.zeros((16, BLOCK_N), dtype=tl.float32)
140
+ for k0 in range(0, K, BLOCK_K):
141
+ offs_k = k0 + tl.arange(0, BLOCK_K)
142
+ w = tl.load(w_ptr + offs_n[:, None] * stride_w + offs_k[None, :],
143
+ mask=(offs_n[:, None] < N) & (offs_k[None, :] < K), other=0.0).to(tl.bfloat16)
144
+ x = tl.load(x_ptr + offs_m[:, None] * stride_x + offs_k[None, :],
145
+ mask=(offs_m[:, None] < M) & (offs_k[None, :] < K), other=0.0)
146
+ acc += tl.dot(x, tl.trans(w))
147
+ acc *= tl.load(s_ptr + offs_n, mask=offs_n < N, other=0.0)[None, :]
148
+ if HAS_BIAS:
149
+ acc += tl.load(b_ptr + offs_n, mask=offs_n < N, other=0.0).to(tl.float32)[None, :]
150
+ tl.store(y_ptr + offs_m[:, None] * stride_y + offs_n[None, :], acc.to(y_ptr.dtype.element_ty),
151
+ mask=(offs_m[:, None] < M) & (offs_n[None, :] < N))
152
+
153
+
154
+ def w8a16_gemv(x: torch.Tensor, weight: torch.Tensor, scale: torch.Tensor, bias: torch.Tensor | None) -> torch.Tensor:
155
+ """y = x @ (weight * scale).T for at most 16 rows of x: FP8 weights dequantized in registers, bandwidth-bound."""
156
+ M, K = x.shape
157
+ N = weight.shape[0]
158
+ x = x.contiguous()
159
+ y = torch.empty((M, N), device=x.device, dtype=x.dtype)
160
+ grid = lambda meta: (triton.cdiv(N, meta["BLOCK_N"]),)
161
+ _w8a16_kernel[grid](x, weight, scale, bias if bias is not None else scale, y, M, N, K, x.stride(0),
162
+ weight.stride(0), y.stride(0), HAS_BIAS=bias is not None)
163
+ return y
164
+
165
+
166
+ class FP8Linear(nn.Module):
167
+ def __init__(self, linear: nn.Linear, fast_accum: bool = False):
168
+ super().__init__()
169
+ weight = linear.weight.detach()
170
+ scale = weight.float().abs().amax(dim=1, keepdim=True).clamp_min(1e-12) / FP8_MAX
171
+ self.register_buffer("weight", (weight.float() / scale).to(FP8).contiguous()) # (N, K)
172
+ self.register_buffer("weight_scale", scale.t().contiguous()) # (1, N)
173
+ self.bias = linear.bias
174
+ self.in_features, self.out_features = linear.in_features, linear.out_features
175
+ self.out_dtype = weight.dtype
176
+ self.fast_accum = fast_accum
177
+
178
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
179
+ shape = x.shape
180
+ rows = x.reshape(-1, shape[-1])
181
+ if rows.shape[0] <= 16: # decode: weight-only, the GEMM is a bandwidth-bound GEMV
182
+ return w8a16_gemv(rows.to(self.out_dtype), self.weight, self.weight_scale.view(-1),
183
+ self.bias).view(*shape[:-1], self.out_features)
184
+ q, scale = rowwise_fp8(rows)
185
+ out = torch._scaled_mm(q, self.weight.t(), scale_a=scale, scale_b=self.weight_scale, bias=self.bias,
186
+ out_dtype=self.out_dtype, use_fast_accum=self.fast_accum)
187
+ return out.view(*shape[:-1], self.out_features)
188
+
189
+
190
+ @torch.no_grad()
191
+ def quantize_fp8(module: nn.Module, fast_accum: bool = False, skip: tuple[str, ...] = ()) -> int:
192
+ """Replace every nn.Linear under `module` (names containing an entry of `skip` excluded) by an FP8Linear."""
193
+ count = 0
194
+ for name, child in list(module.named_modules()):
195
+ for child_name, leaf in list(child.named_children()):
196
+ full = f"{name}.{child_name}" if name else child_name
197
+ if isinstance(leaf, nn.Linear) and not any(s in full for s in skip):
198
+ setattr(child, child_name, FP8Linear(leaf, fast_accum))
199
+ del leaf
200
+ count += 1
201
+ _free_cache()
202
+ return count
203
+
204
+
205
+ # CUDA-graph greedy decoding -----------------------------------------------------------------------------------------
206
+
207
+
208
+ class GraphedGreedyGenerate:
209
+ """Greedy decoding of one sequence for a Qwen3_5ForConditionalGeneration, with the decode step replayed as a CUDA
210
+ graph. A drop-in for `model.generate(**inputs, max_new_tokens=..., do_sample=False, eos_token_id=...)`.
211
+
212
+ The prefill runs eagerly into a persistent StaticCache (its linear-attention states and KV tensors are updated in
213
+ place, so their addresses never change). One decode step (embed, 32 layers, lm_head, argmax) is captured once over
214
+ static buffers: the input token, its absolute position, the M-RoPE offset and the attention mask over the cache.
215
+ Per token only the graph is replayed; the end token is checked every `check_every` tokens.
216
+ """
217
+
218
+ def __init__(self, model, max_cache_len: int = 12288, max_new_tokens: int = 4096, check_every: int = 16):
219
+ from transformers import StaticCache
220
+
221
+ self.model, self.generate_eager = model, model.generate
222
+ self.max_cache_len, self.check_every = max_cache_len, check_every
223
+ device = model.device
224
+ self.cache = StaticCache(config=model.config, max_cache_len=max_cache_len)
225
+ self.tok = torch.zeros((1, 1), dtype=torch.long, device=device)
226
+ self.pos = torch.zeros((1,), dtype=torch.long, device=device)
227
+ self.offset = torch.zeros((4, 1, 1), dtype=torch.long, device=device)
228
+ self.mask = torch.zeros((1, 1, 1, max_cache_len), dtype=torch.bool, device=device)
229
+ self.out = torch.zeros((max_new_tokens,), dtype=torch.long, device=device)
230
+ self.count = torch.zeros((1,), dtype=torch.long, device=device)
231
+ self.graph = None
232
+
233
+ def _step(self):
234
+ text_model = self.model.model.language_model
235
+ self.mask.index_fill_(3, self.pos, True)
236
+ hidden = text_model(
237
+ inputs_embeds=text_model.embed_tokens(self.tok),
238
+ position_ids=self.pos.view(1, 1, 1) + self.offset,
239
+ attention_mask={"full_attention": self.mask, "linear_attention": None},
240
+ past_key_values=self.cache,
241
+ use_cache=True,
242
+ ).last_hidden_state
243
+ token = self.model.lm_head(hidden[:, -1]).argmax(-1)
244
+ self.out.index_copy_(0, self.count, token)
245
+ self.tok.copy_(token.view(1, 1))
246
+ self.pos.add_(1)
247
+ self.count.add_(1)
248
+
249
+ def _prefill(self, input_ids, kwargs):
250
+ self.cache.reset()
251
+ self.mask.zero_()
252
+ self.model.model.rope_deltas = None
253
+ logits = self.model(input_ids=input_ids, past_key_values=self.cache, use_cache=True, logits_to_keep=1,
254
+ **kwargs).logits
255
+ token = logits[:, -1].argmax(-1)
256
+ rope_deltas = self.model.model.rope_deltas
257
+ self.offset.zero_()
258
+ if rope_deltas is not None:
259
+ self.offset[1:] = rope_deltas.view(-1)[0]
260
+ prompt_length = input_ids.shape[1]
261
+ self.mask[..., :prompt_length] = True
262
+ self.out[0] = token[0]
263
+ self.count.fill_(1)
264
+ self.tok.copy_(token.view(1, 1))
265
+ self.pos.fill_(prompt_length)
266
+
267
+ def _capture(self):
268
+ stream = torch.cuda.Stream()
269
+ stream.wait_stream(torch.cuda.current_stream())
270
+ with torch.cuda.stream(stream):
271
+ for _ in range(3):
272
+ self._step()
273
+ torch.cuda.current_stream().wait_stream(stream)
274
+ self.graph = torch.cuda.CUDAGraph()
275
+ with torch.cuda.graph(self.graph):
276
+ self._step()
277
+
278
+ @torch.no_grad()
279
+ def generate(self, input_ids, max_new_tokens: int, eos_token_id: int, **kwargs):
280
+ kwargs.pop("do_sample", None)
281
+ kwargs.pop("pad_token_id", None)
282
+ if input_ids.shape[0] != 1 or input_ids.shape[1] + max_new_tokens > self.max_cache_len:
283
+ return self.generate_eager(input_ids=input_ids, max_new_tokens=max_new_tokens, do_sample=False,
284
+ eos_token_id=eos_token_id, **kwargs)
285
+ if self.graph is None:
286
+ self._prefill(input_ids, kwargs)
287
+ self._capture()
288
+ self._prefill(input_ids, kwargs)
289
+ generated = 1
290
+ while generated < max_new_tokens and not bool((self.out[:generated] == eos_token_id).any()):
291
+ for _ in range(min(self.check_every, max_new_tokens - generated)):
292
+ self.graph.replay()
293
+ generated = min(generated + self.check_every, max_new_tokens)
294
+ tokens = self.out[:generated]
295
+ ends = (tokens == eos_token_id).nonzero()
296
+ if ends.numel():
297
+ tokens = tokens[: int(ends[0, 0]) + 1]
298
+ return torch.cat([input_ids, tokens[None].to(input_ids.device)], dim=1)
299
+
300
+
301
+ # MTP speculative decoding -------------------------------------------------------------------------------------------
302
+
303
+
304
+ class MTPHead(nn.Module):
305
+ """Qwen3.5's multi-token-prediction head: (hidden state at i, embedding of token i+1) -> hidden for token i+2."""
306
+
307
+ def __init__(self, config):
308
+ super().__init__()
309
+ from transformers.models.qwen3_5.modeling_qwen3_5 import Qwen3_5DecoderLayer, Qwen3_5RMSNorm
310
+
311
+ size, eps = config.hidden_size, config.rms_norm_eps
312
+ self.config = config
313
+ self.fc = nn.Linear(2 * size, size, bias=False)
314
+ self.pre_fc_norm_embedding = Qwen3_5RMSNorm(size, eps)
315
+ self.pre_fc_norm_hidden = Qwen3_5RMSNorm(size, eps)
316
+ self.layers = nn.ModuleList([Qwen3_5DecoderLayer(config, 0)])
317
+ self.norm = Qwen3_5RMSNorm(size, eps)
318
+
319
+ def forward(self, embeds, hidden, position_embeddings, text_positions, mask, cache):
320
+ x = self.fc(torch.cat([self.pre_fc_norm_embedding(embeds), self.pre_fc_norm_hidden(hidden)], dim=-1))
321
+ x = self.layers[0](x, position_embeddings=position_embeddings, attention_mask=mask, position_ids=text_positions,
322
+ past_key_values=cache, use_cache=True)
323
+ return self.norm(x)
324
+
325
+
326
+ MTP_REPO, MTP_REVISION = "Qwen/Qwen3.5-9B", "c202236235762e1c871ad0ccb60c8ee5ba337b9a"
327
+
328
+
329
+ def read_hub_tensors(repo: str, revision: str, prefix: str) -> dict[str, torch.Tensor]:
330
+ """The tensors named `prefix*` of a sharded safetensors repo, read by byte range (no full shard downloads)."""
331
+ import json
332
+ import struct
333
+
334
+ from huggingface_hub import HfFileSystem, hf_hub_download
335
+
336
+ dtypes = {"BF16": torch.bfloat16, "F16": torch.float16, "F32": torch.float32}
337
+ index = json.load(open(hf_hub_download(repo, "model.safetensors.index.json", revision=revision)))
338
+ fs, tensors = HfFileSystem(), {}
339
+ for file in sorted({f for name, f in index["weight_map"].items() if name.startswith(prefix)}):
340
+ with fs.open(f"{repo}/{file}", "rb", revision=revision) as handle:
341
+ header_size = struct.unpack("<Q", handle.read(8))[0]
342
+ header = json.loads(handle.read(header_size))
343
+ for name, meta in header.items():
344
+ if name.startswith(prefix):
345
+ start, end = meta["data_offsets"]
346
+ handle.seek(8 + header_size + start)
347
+ data = bytearray(handle.read(end - start))
348
+ tensors[name] = torch.frombuffer(data, dtype=dtypes[meta["dtype"]]).reshape(meta["shape"])
349
+ return tensors
350
+
351
+
352
+ def load_mtp_head(model, repo: str = MTP_REPO, revision: str = MTP_REVISION) -> MTPHead:
353
+ """Qwen3.5-9B's MTP head (transformers drops it), used as the prompt upsampler's draft model."""
354
+ import copy
355
+
356
+ config = copy.deepcopy(model.config.get_text_config())
357
+ config.num_hidden_layers, config.layer_types = 1, ["full_attention"]
358
+ state = {name[len("mtp."):]: tensor for name, tensor in read_hub_tensors(repo, revision, "mtp.").items()}
359
+ head = MTPHead(config)
360
+ head.load_state_dict(state)
361
+ return head.to(model.device, model.dtype).eval()
362
+
363
+
364
+ GDN_VERIFY = {"torch_reference": False}
365
+
366
+
367
+ def _gdn_verify(module, hidden_states, cache_params):
368
+ """Qwen3_5GatedDeltaNet over a few cached tokens, one token at a time, keeping the conv and recurrent states after
369
+ every token in `module._spec_snaps` so a rejected draft rolls back by index."""
370
+ from torch.nn import functional as F
371
+ from transformers.models.qwen3_5 import modeling_qwen3_5 as qm
372
+
373
+ batch, length, _ = hidden_states.shape
374
+ layer = cache_params.layers[module.layer_idx]
375
+ conv_state, recurrent_state = layer.conv_states[0], layer.recurrent_states[0]
376
+ conv_snaps, recurrent_snaps = module._spec_snaps
377
+ mixed = module.in_proj_qkv(hidden_states).transpose(1, 2)
378
+ z = module.in_proj_z(hidden_states).reshape(batch, length, -1, module.head_v_dim)
379
+ b, a = module.in_proj_b(hidden_states), module.in_proj_a(hidden_states)
380
+ convolved = []
381
+ for j in range(length):
382
+ convolved.append(qm.causal_conv1d_update(mixed[:, :, j : j + 1], conv_state, module.conv1d.weight.squeeze(1),
383
+ module.conv1d.bias, module.activation))
384
+ conv_snaps[j].copy_(conv_state)
385
+ mixed = torch.cat(convolved, dim=2).transpose(1, 2)
386
+ query, key, value = torch.split(mixed, [module.key_dim, module.key_dim, module.value_dim], dim=-1)
387
+ query = query.reshape(batch, length, -1, module.head_k_dim)
388
+ key = key.reshape(batch, length, -1, module.head_k_dim)
389
+ value = value.reshape(batch, length, -1, module.head_v_dim)
390
+ beta = b.sigmoid()
391
+ g = -module.A_log.float().exp() * F.softplus(a.float() + module.dt_bias)
392
+ if module.num_v_heads // module.num_k_heads > 1:
393
+ query = query.repeat_interleave(module.num_v_heads // module.num_k_heads, dim=2)
394
+ key = key.repeat_interleave(module.num_v_heads // module.num_k_heads, dim=2)
395
+ recurrence = qm.torch_recurrent_gated_delta_rule
396
+ if GDN_VERIFY["torch_reference"]:
397
+ recurrence = getattr(recurrence, "__wrapped__", recurrence)
398
+ outputs = []
399
+ for j in range(length):
400
+ step = [t[:, j : j + 1].contiguous() for t in (query, key, value, g, beta)]
401
+ out, state = recurrence(step[0], step[1], step[2], g=step[3], beta=step[4],
402
+ initial_state=recurrent_state.clone(), output_final_state=True,
403
+ use_qk_l2norm_in_kernel=True)
404
+ recurrent_state.copy_(state)
405
+ recurrent_snaps[j].copy_(recurrent_state)
406
+ outputs.append(out)
407
+ core = module.norm(torch.cat(outputs, dim=1).reshape(-1, module.head_v_dim), z.reshape(-1, module.head_v_dim))
408
+ return module.out_proj(core.reshape(batch, length, -1))
409
+
410
+
411
+ def _install_gdn_verify(module) -> None:
412
+ """Route `module`'s forward through `_gdn_verify` while `module._spec_verify` is set. Patched on the instance:
413
+ `set_use_kernels` installs instance-level forwards, which a class-level patch would not reach."""
414
+ if getattr(module, "_spec_patched", False):
415
+ return
416
+ original = module.forward
417
+
418
+ def forward(hidden_states, cache_params=None, attention_mask=None, **kwargs):
419
+ if module._spec_verify:
420
+ return _gdn_verify(module, hidden_states, cache_params)
421
+ return original(hidden_states, cache_params=cache_params, attention_mask=attention_mask, **kwargs)
422
+
423
+ module.forward, module._spec_verify, module._spec_patched = forward, False, True
424
+
425
+
426
+ class MTPGreedyGenerate:
427
+ """Greedy decoding of one sequence with one MTP draft token per step, each step one CUDA graph. Lossless: every
428
+ emitted token is the main model's own greedy choice; an accepted draft adds a second token to the step.
429
+
430
+ A step feeds `[t, draft]` to the main model (its Gated DeltaNet layers keep their states after each token, the
431
+ full-attention layers write both K/V rows), takes the greedy predictions p0, p1, accepts the draft if it equals
432
+ p0, rolls the states back to the accepted prefix and emits `draft, p1` or `p0`. The MTP head then runs on the two
433
+ (hidden, next token) pairs of the step and drafts from the accepted one. The end token is checked every
434
+ `check_every` steps.
435
+ """
436
+
437
+ def __init__(self, model, mtp: MTPHead, max_cache_len: int = 12288, max_new_tokens: int = 4096,
438
+ check_every: int = 8):
439
+ from transformers import StaticCache
440
+
441
+ self.model, self.mtp, self.generate_eager = model, mtp, model.generate
442
+ self.text_model = model.model.language_model
443
+ self.max_cache_len, self.check_every = max_cache_len, check_every
444
+ device = model.device
445
+ self.cache = StaticCache(config=model.config, max_cache_len=max_cache_len)
446
+ self.mtp_cache = StaticCache(config=mtp.config, max_cache_len=max_cache_len)
447
+ self.gdn = [layer.linear_attn for layer in self.text_model.layers if layer.block_type == "linear_attention"]
448
+ for module in self.gdn:
449
+ _install_gdn_verify(module)
450
+ long = dict(dtype=torch.long, device=device)
451
+ self.tok, self.draft, self.pos = (torch.zeros((1,), **long) for _ in range(3))
452
+ self.offset = torch.zeros((4, 1, 1), **long)
453
+ self.keys = torch.arange(max_cache_len, **long)
454
+ self.pair = torch.arange(2, **long)
455
+ self.out = torch.zeros((max_new_tokens + 2 * check_every + 2,), **long)
456
+ self.count, self.accepted = torch.zeros((1,), **long), torch.zeros((1,), **long)
457
+ self.max_new_tokens = max_new_tokens
458
+ self.graph, self.stats = None, {}
459
+
460
+ def _set_verify(self, on: bool):
461
+ for module in self.gdn:
462
+ module._spec_verify = on
463
+
464
+ def _static_lengths(self):
465
+ from transformers.cache_utils import StaticLayer
466
+
467
+ return [layer.cumulative_length for cache in (self.cache, self.mtp_cache) for layer in cache.layers
468
+ if isinstance(layer, StaticLayer)]
469
+
470
+ def _step(self):
471
+ positions = self.pos + self.pair
472
+ position_ids = positions.view(1, 1, 2) + self.offset
473
+ mask = (self.keys.view(1, -1) <= positions.view(-1, 1)).view(1, 1, 2, -1)
474
+ masks = {"full_attention": mask, "linear_attention": None}
475
+ tokens = torch.cat([self.tok, self.draft]).view(1, 2)
476
+ hidden = self.text_model(inputs_embeds=self.text_model.embed_tokens(tokens), position_ids=position_ids,
477
+ attention_mask=masks, past_key_values=self.cache, use_cache=True).last_hidden_state
478
+ preds = self.model.lm_head(hidden).argmax(-1).view(2)
479
+ accept = (preds[0] == self.draft[0]).long().view(1)
480
+ for module in self.gdn:
481
+ layer = self.cache.layers[module.layer_idx]
482
+ conv_snaps, recurrent_snaps = module._spec_snaps
483
+ layer.conv_states[0].copy_(conv_snaps.index_select(0, accept)[0])
484
+ layer.recurrent_states[0].copy_(recurrent_snaps.index_select(0, accept)[0])
485
+ emitted = torch.stack([torch.where(accept.bool(), self.draft, preds[:1])[0], preds[1]])
486
+ self.out.index_copy_(0, self.count + self.pair, emitted)
487
+ cos_sin = self.text_model.rotary_emb(hidden, position_ids[1:])
488
+ mtp_hidden = self.mtp(self.text_model.embed_tokens(emitted.view(1, 2)), hidden, cos_sin, position_ids[0], mask,
489
+ self.mtp_cache)
490
+ draft = self.model.lm_head(mtp_hidden.index_select(1, accept)).argmax(-1).view(1)
491
+ new_length = (self.pos + 1 + accept).view(())
492
+ for length in self._static_lengths():
493
+ length.copy_(new_length)
494
+ self.tok.copy_(torch.where(accept.bool(), preds[1:], preds[:1]))
495
+ self.draft.copy_(draft)
496
+ self.pos.add_(1 + accept)
497
+ self.count.add_(1 + accept)
498
+ self.accepted.add_(accept)
499
+
500
+ def _prefill(self, input_ids, kwargs):
501
+ self.cache.reset()
502
+ self.mtp_cache.reset()
503
+ self._set_verify(False)
504
+ self.model.model.rope_deltas = None
505
+ seen = {}
506
+ hook = self.text_model.register_forward_pre_hook(
507
+ lambda module, args, kw: seen.update(position_ids=kw.get("position_ids")), with_kwargs=True)
508
+ hidden = self.model.model(input_ids=input_ids, past_key_values=self.cache, use_cache=True,
509
+ **kwargs).last_hidden_state
510
+ hook.remove()
511
+ first = self.model.lm_head(hidden[:, -1]).argmax(-1)
512
+ length = input_ids.shape[1]
513
+ steps = torch.arange(length, device=input_ids.device)
514
+ rope = seen.get("position_ids")
515
+ rope = steps.view(1, 1, -1).expand(3, 1, -1) if rope is None else rope[-3:]
516
+ mask = (self.keys.view(1, -1) <= steps.view(-1, 1)).view(1, 1, length, -1)
517
+ next_ids = torch.cat([input_ids[:, 1:], first.view(1, 1)], dim=1)
518
+ mtp_hidden = self.mtp(self.text_model.embed_tokens(next_ids), hidden, self.text_model.rotary_emb(hidden, rope),
519
+ steps.view(1, -1), mask, self.mtp_cache)
520
+ draft = self.model.lm_head(mtp_hidden[:, -1]).argmax(-1)
521
+ rope_deltas = self.model.model.rope_deltas
522
+ self.offset.zero_()
523
+ if rope_deltas is not None:
524
+ self.offset[1:] = rope_deltas.view(-1)[0]
525
+ self.out[0] = first[0]
526
+ self.count.fill_(1)
527
+ self.accepted.zero_()
528
+ self.tok.copy_(first)
529
+ self.draft.copy_(draft)
530
+ self.pos.fill_(length)
531
+ self._set_verify(True)
532
+
533
+ def _capture(self):
534
+ for module in self.gdn:
535
+ layer = self.cache.layers[module.layer_idx]
536
+ module._spec_snaps = (torch.empty((2, *layer.conv_states[0].shape), dtype=layer.conv_states[0].dtype,
537
+ device=layer.conv_states[0].device),
538
+ torch.empty((2, *layer.recurrent_states[0].shape),
539
+ dtype=layer.recurrent_states[0].dtype,
540
+ device=layer.recurrent_states[0].device))
541
+ stream = torch.cuda.Stream()
542
+ stream.wait_stream(torch.cuda.current_stream())
543
+ with torch.cuda.stream(stream):
544
+ for _ in range(3):
545
+ self._step()
546
+ torch.cuda.current_stream().wait_stream(stream)
547
+ self.graph = torch.cuda.CUDAGraph()
548
+ with torch.cuda.graph(self.graph):
549
+ self._step()
550
+
551
+ @torch.no_grad()
552
+ def generate(self, input_ids, max_new_tokens: int, eos_token_id: int, **kwargs):
553
+ kwargs.pop("do_sample", None)
554
+ kwargs.pop("pad_token_id", None)
555
+ margin = 2 * self.check_every + 2
556
+ self._set_verify(False)
557
+ if (input_ids.shape[0] != 1 or max_new_tokens > self.max_new_tokens
558
+ or input_ids.shape[1] + max_new_tokens + margin > self.max_cache_len):
559
+ return self.generate_eager(input_ids=input_ids, max_new_tokens=max_new_tokens, do_sample=False,
560
+ eos_token_id=eos_token_id, **kwargs)
561
+ if self.graph is None:
562
+ self._prefill(input_ids, kwargs)
563
+ self._capture()
564
+ self._prefill(input_ids, kwargs)
565
+ steps, count = 0, 1
566
+ while count < max_new_tokens and not bool((self.out[:count] == eos_token_id).any()):
567
+ for _ in range(self.check_every):
568
+ self.graph.replay()
569
+ steps += self.check_every
570
+ count = int(self.count)
571
+ tokens = self.out[: min(count, max_new_tokens)]
572
+ ends = (tokens == eos_token_id).nonzero()
573
+ if ends.numel():
574
+ tokens = tokens[: int(ends[0, 0]) + 1]
575
+ self._set_verify(False)
576
+ self.stats = {"steps": steps, "tokens": int(tokens.numel()), "accepted": int(self.accepted),
577
+ "acceptance": round(int(self.accepted) / max(steps, 1), 3)}
578
+ return torch.cat([input_ids, tokens[None].to(input_ids.device)], dim=1)
579
+
580
+
581
+ # Decode attention ---------------------------------------------------------------------------------------------------
582
+
583
+
584
+ def install_decode_attention(*configs) -> None:
585
+ """Full attention for a few query rows over a long static cache as two grouped batched GEMVs (K/V read once per
586
+ KV head, no repeat). SDPA's memory-efficient kernel parallelizes a single query poorly. Longer queries (prefill)
587
+ keep SDPA."""
588
+ from transformers import AttentionInterface
589
+ from transformers.integrations.sdpa_attention import sdpa_attention_forward
590
+
591
+ def attention(module, query, key, value, attention_mask, dropout=0.0, scaling=None, **kwargs):
592
+ batch, heads, length, dim = query.shape
593
+ if length > 16 or attention_mask is None or attention_mask.dtype != torch.bool:
594
+ return sdpa_attention_forward(module, query, key, value, attention_mask, dropout=dropout,
595
+ scaling=scaling, **kwargs)
596
+ kv_heads, keys = key.shape[1], key.shape[2]
597
+ groups = heads // kv_heads
598
+ q = query.reshape(batch, kv_heads, groups * length, dim)
599
+ scores = torch.matmul(q, key.transpose(-1, -2)).float().view(batch, kv_heads, groups, length, keys)
600
+ scores = scores * (scaling if scaling is not None else dim**-0.5)
601
+ scores = scores.masked_fill(~attention_mask[..., :keys].view(1, 1, 1, length, keys), float("-inf"))
602
+ probs = scores.softmax(-1).to(value.dtype).view(batch, kv_heads, groups * length, keys)
603
+ out = torch.matmul(probs, value).view(batch, heads, length, -1)
604
+ return out.transpose(1, 2).contiguous(), None
605
+
606
+ AttentionInterface.register("p_h3_decode", attention)
607
+ for config in configs:
608
+ config._attn_implementation = "p_h3_decode"
609
+
610
+
611
+ # Batched VAE tile decode --------------------------------------------------------------------------------------------
612
+
613
+
614
+ def batch_vae_tiles(vae, max_tiles: int = 96) -> None:
615
+ """Decode the spatial tiles of `AutoencoderKLPH3Image` in batches of up to `max_tiles` instead of one decoder call
616
+ per tile. The tile layout, the per-tile decoder math and the stitching are unchanged."""
617
+
618
+ def decode_clip(z: torch.Tensor) -> torch.Tensor:
619
+ if not vae.use_tiling:
620
+ return vae.decoder(vae.post_quant_conv(z))
621
+ ratio = vae.spatial_compression_ratio
622
+ y_indices, y_lengths, y_overlaps = vae._split_tiles(
623
+ z.shape[-2] * ratio, vae.tile_sample_min_height, vae.tile_sample_min_overlap_height)
624
+ x_indices, x_lengths, x_overlaps = vae._split_tiles(
625
+ z.shape[-1] * ratio, vae.tile_sample_min_width, vae.tile_sample_min_overlap_width)
626
+ tiles = {}
627
+ for i, (i_pos, i_len) in enumerate(zip(y_indices, y_lengths)):
628
+ for j, (j_pos, j_len) in enumerate(zip(x_indices, x_lengths)):
629
+ tiles[(i, j)] = z[..., i_pos // ratio: i_pos // ratio + i_len // ratio,
630
+ j_pos // ratio: j_pos // ratio + j_len // ratio]
631
+ decoded = {}
632
+ by_shape: dict[tuple, list] = {}
633
+ for key, tile in tiles.items():
634
+ by_shape.setdefault(tuple(tile.shape), []).append(key)
635
+ for keys in by_shape.values():
636
+ for start in range(0, len(keys), max_tiles):
637
+ chunk = keys[start: start + max_tiles]
638
+ out = vae.decoder(vae.post_quant_conv(torch.cat([tiles[k] for k in chunk], dim=0)))
639
+ for index, key in enumerate(chunk):
640
+ decoded[key] = out[index: index + 1]
641
+ rows = [[decoded[(i, j)] for j in range(len(x_indices))] for i in range(len(y_indices))]
642
+ return vae._stitch_tiles(rows, y_overlaps, x_overlaps)
643
+
644
+ vae._decode_clip = decode_clip
645
+
646
+
647
+ # The Space's pipeline ------------------------------------------------------------------------------------------------
648
+
649
+
650
+ def load_slim_pipeline(path: str, adaln_calls: list, device: str = "cuda", log=print):
651
+ """p-h3-image from a local snapshot, slimmed for a 48 GB slice. Components are loaded and slimmed one at a time
652
+ (bf16 transformer peak ~66 GB host RAM) and moved to `device`. Returns the pipeline and the upsampler's MTP head."""
653
+ import gc
654
+
655
+ from diffusers import ModularPipeline
656
+
657
+ start = time.time()
658
+ pipe = ModularPipeline.from_pretrained(path)
659
+
660
+ pipe.load_components(names=["transformer"], dtype=torch.bfloat16)
661
+ install_adaln_tables(pipe.transformer, [torch.tensor(call) for call in adaln_calls])
662
+ quantize_fp8(pipe.transformer.transformer_blocks)
663
+ fuse_dit_blocks(pipe.transformer)
664
+ pipe.transformer.to(device)
665
+ gc.collect()
666
+ log(f"[{time.time() - start:6.1f}s] transformer ready")
667
+
668
+ pipe.load_components(names=["prompt_upsampler", "prompt_upsampler_processor"], dtype=torch.bfloat16)
669
+ upsampler = pipe.prompt_upsampler
670
+ quantize_fp8(upsampler.model.language_model)
671
+ upsampler.lm_head = FP8Linear(upsampler.lm_head)
672
+ mtp = load_mtp_head(upsampler)
673
+ quantize_fp8(mtp)
674
+ install_decode_attention(upsampler.model.language_model.config, mtp.config)
675
+ upsampler.to(device)
676
+ mtp.to(device)
677
+ gc.collect()
678
+ log(f"[{time.time() - start:6.1f}s] prompt upsampler ready")
679
+
680
+ pipe.load_components(dtype=torch.bfloat16)
681
+ quantize_fp8(pipe.text_encoder.model.language_model)
682
+ quantize_fp8(pipe.vae.decoder.transformer_blocks)
683
+ batch_vae_tiles(pipe.vae)
684
+ pipe.to(device)
685
+ gc.collect()
686
+ log(f"[{time.time() - start:6.1f}s] pipeline ready")
687
+ return pipe, mtp
688
+
689
+
690
+ # Fused DiT block ----------------------------------------------------------------------------------------------------
691
+ #
692
+ # One MiniMaxH3TransformerBlock in five row kernels around the FP8 GEMMs and attention. Every producer of a GEMM input
693
+ # holds the whole row, so it also computes the row's FP8 scale and writes e4m3 directly (no separate quant pass):
694
+ # norm1 + AdaLN modulation -> FP8 | QKV GEMM | q/k RMSNorm + RoPE | attention | out GEMM
695
+ # gated residual + norm2 + modulation -> FP8 | FF-in GEMM | SwiGLU -> FP8 | FF-out GEMM | gated residual
696
+ # Math runs in fp32 and rounds once per output (eager rounds to bf16 after every op).
697
+
698
+
699
+ @triton.jit
700
+ def _rms_mod_fp8_kernel(x_ptr, w_ptr, scale_ptr, shift_ptr, idx_ptr, q_ptr, s_ptr, n_cols, stride_x, stride_tab, eps,
701
+ BLOCK: tl.constexpr):
702
+ row = tl.program_id(0).to(tl.int64)
703
+ cols = tl.arange(0, BLOCK)
704
+ mask = cols < n_cols
705
+ x = tl.load(x_ptr + row * stride_x + cols, mask=mask, other=0.0).to(tl.float32)
706
+ w = tl.load(w_ptr + cols, mask=mask, other=0.0).to(tl.float32)
707
+ tab = tl.load(idx_ptr + row).to(tl.int64) * stride_tab + cols
708
+ sc = tl.load(scale_ptr + tab, mask=mask, other=0.0).to(tl.float32)
709
+ sh = tl.load(shift_ptr + tab, mask=mask, other=0.0).to(tl.float32)
710
+ y = x * tl.math.rsqrt(tl.sum(x * x, axis=0) / n_cols + eps) * w * (1.0 + sc) + sh
711
+ amax = tl.maximum(tl.max(tl.where(mask, tl.abs(y), 0.0), axis=0), 1e-12)
712
+ tl.store(q_ptr + row * n_cols + cols, (y * (448.0 / amax)).to(tl.float8e4nv), mask=mask)
713
+ tl.store(s_ptr + row, amax / 448.0)
714
+
715
+
716
+ @triton.jit
717
+ def _res_gate_rms_mod_fp8_kernel(h_ptr, b_ptr, gate_ptr, w_ptr, scale_ptr, shift_ptr, idx_ptr, out_ptr, q_ptr, s_ptr,
718
+ n_cols, stride_h, stride_b, stride_tab, eps, BLOCK: tl.constexpr):
719
+ row = tl.program_id(0).to(tl.int64)
720
+ cols = tl.arange(0, BLOCK)
721
+ mask = cols < n_cols
722
+ tab = tl.load(idx_ptr + row).to(tl.int64) * stride_tab + cols
723
+ h = tl.load(h_ptr + row * stride_h + cols, mask=mask, other=0.0).to(tl.float32)
724
+ b = tl.load(b_ptr + row * stride_b + cols, mask=mask, other=0.0).to(tl.float32)
725
+ g = tl.load(gate_ptr + tab, mask=mask, other=0.0).to(tl.float32)
726
+ h = (h + g * b).to(out_ptr.dtype.element_ty)
727
+ tl.store(out_ptr + row * n_cols + cols, h, mask=mask)
728
+ h = h.to(tl.float32)
729
+ w = tl.load(w_ptr + cols, mask=mask, other=0.0).to(tl.float32)
730
+ sc = tl.load(scale_ptr + tab, mask=mask, other=0.0).to(tl.float32)
731
+ sh = tl.load(shift_ptr + tab, mask=mask, other=0.0).to(tl.float32)
732
+ y = h * tl.math.rsqrt(tl.sum(h * h, axis=0) / n_cols + eps) * w * (1.0 + sc) + sh
733
+ amax = tl.maximum(tl.max(tl.where(mask, tl.abs(y), 0.0), axis=0), 1e-12)
734
+ tl.store(q_ptr + row * n_cols + cols, (y * (448.0 / amax)).to(tl.float8e4nv), mask=mask)
735
+ tl.store(s_ptr + row, amax / 448.0)
736
+
737
+
738
+ @triton.jit
739
+ def _res_gate_kernel(h_ptr, b_ptr, gate_ptr, idx_ptr, out_ptr, n_cols, stride_h, stride_b, stride_tab,
740
+ BLOCK: tl.constexpr):
741
+ row = tl.program_id(0).to(tl.int64)
742
+ cols = tl.arange(0, BLOCK)
743
+ mask = cols < n_cols
744
+ tab = tl.load(idx_ptr + row).to(tl.int64) * stride_tab + cols
745
+ h = tl.load(h_ptr + row * stride_h + cols, mask=mask, other=0.0).to(tl.float32)
746
+ b = tl.load(b_ptr + row * stride_b + cols, mask=mask, other=0.0).to(tl.float32)
747
+ g = tl.load(gate_ptr + tab, mask=mask, other=0.0).to(tl.float32)
748
+ tl.store(out_ptr + row * n_cols + cols, (h + g * b).to(out_ptr.dtype.element_ty), mask=mask)
749
+
750
+
751
+ @triton.jit
752
+ def _swiglu_fp8_kernel(x_ptr, q_ptr, s_ptr, n_cols, stride_x, BLOCK: tl.constexpr):
753
+ row = tl.program_id(0).to(tl.int64)
754
+ cols = tl.arange(0, BLOCK)
755
+ mask = cols < n_cols
756
+ v = tl.load(x_ptr + row * stride_x + cols, mask=mask, other=0.0).to(tl.float32)
757
+ g = tl.load(x_ptr + row * stride_x + n_cols + cols, mask=mask, other=0.0).to(tl.float32)
758
+ y = v * g * tl.sigmoid(g)
759
+ amax = tl.maximum(tl.max(tl.where(mask, tl.abs(y), 0.0), axis=0), 1e-12)
760
+ tl.store(q_ptr + row * n_cols + cols, (y * (448.0 / amax)).to(tl.float8e4nv), mask=mask)
761
+ tl.store(s_ptr + row, amax / 448.0)
762
+
763
+
764
+ @triton.jit
765
+ def _qk_norm_rope_kernel(qkv_ptr, wq_ptr, wk_ptr, cos_ptr, sin_ptr, q_ptr, k_ptr, stride_qkv, stride_rope, heads,
766
+ eps, D: tl.constexpr, ROT: tl.constexpr):
767
+ row = tl.program_id(0).to(tl.int64)
768
+ which = tl.program_id(1) # [0, heads) query heads, [heads, 2 * heads) key heads
769
+ is_k = which >= heads
770
+ head = which - heads * is_k.to(tl.int32)
771
+ c = tl.arange(0, D)
772
+ half = ROT // 2
773
+ partner = tl.where(c < half, c + half, tl.where(c < ROT, c - half, c))
774
+ base = qkv_ptr + row * stride_qkv + (is_k.to(tl.int64) * heads + head) * D
775
+ x = tl.load(base + c).to(tl.float32)
776
+ xp = tl.load(base + partner).to(tl.float32)
777
+ w = tl.where(is_k, tl.load(wk_ptr + c), tl.load(wq_ptr + c)).to(tl.float32)
778
+ wp = tl.where(is_k, tl.load(wk_ptr + partner), tl.load(wq_ptr + partner)).to(tl.float32)
779
+ r = tl.math.rsqrt(tl.sum(x * x, axis=0) / D + eps)
780
+ xn, xpn = x * r * w, xp * r * wp
781
+ rot = c < ROT
782
+ cos = tl.load(cos_ptr + row * stride_rope + c, mask=rot, other=1.0)
783
+ sin = tl.load(sin_ptr + row * stride_rope + c, mask=rot, other=0.0)
784
+ y = (xn * cos + tl.where(c < half, -xpn, xpn) * sin).to(q_ptr.dtype.element_ty)
785
+ offset = (row * heads + head) * D + c
786
+ tl.store(q_ptr + offset, y, mask=(c < D) & (which < heads))
787
+ tl.store(k_ptr + offset, y, mask=(c < D) & (which >= heads))
788
+
789
+
790
+ def _row_launch(kernel, rows, cols, *args, **kwargs):
791
+ block = triton.next_power_of_2(cols)
792
+ kernel[(rows,)](*args, BLOCK=block, num_warps=16 if block >= 16384 else 8, **kwargs)
793
+
794
+
795
+ def _fp8_gemm(q, s, linear: FP8Linear):
796
+ return torch._scaled_mm(q, linear.weight.t(), scale_a=s, scale_b=linear.weight_scale, bias=linear.bias,
797
+ out_dtype=linear.out_dtype, use_fast_accum=linear.fast_accum)
798
+
799
+
800
+ def _fused_block_forward(block, hidden_states, temb, adaln_indices, rotary_emb, attention_mask=None):
801
+ from diffusers.models.attention_dispatch import dispatch_attention_fn
802
+
803
+ shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = block.adaln_proj(temb)
804
+ batch, rows, cols = hidden_states.shape
805
+ h = hidden_states.reshape(rows, cols)
806
+ idx, stride_tab, device = adaln_indices, shift_msa.stride(0), h.device
807
+ attn = block.attn
808
+ heads, dim = attn.heads, attn.head_dim
809
+ q8 = torch.empty((rows, cols), device=device, dtype=FP8)
810
+ s = torch.empty((rows, 1), device=device, dtype=torch.float32)
811
+ _row_launch(_rms_mod_fp8_kernel, rows, cols, h, block.norm1.weight, scale_msa, shift_msa, idx, q8, s, cols,
812
+ h.stride(0), stride_tab, block.norm1.eps)
813
+ qkv = _fp8_gemm(q8, s, block._fused_qkv)
814
+ q = torch.empty((rows, heads, dim), device=device, dtype=h.dtype)
815
+ k = torch.empty_like(q)
816
+ cos, sin = rotary_emb
817
+ _qk_norm_rope_kernel[(rows, 2 * heads)](qkv, attn.norm_q.weight, attn.norm_k.weight, cos, sin, q, k, qkv.stride(0),
818
+ cos.stride(0), heads, attn.norm_q.eps, D=dim, ROT=cos.shape[-1],
819
+ num_warps=2)
820
+ v = qkv[:, 2 * heads * dim:].view(rows, heads, dim)
821
+ o = dispatch_attention_fn(q[None], k[None], v[None], attn_mask=None, dropout_p=0.0, is_causal=False,
822
+ backend=attn.processor._attention_backend)
823
+ attn_out = attn.to_out[0](o.reshape(rows, heads * dim))
824
+ h2 = torch.empty_like(h)
825
+ _row_launch(_res_gate_rms_mod_fp8_kernel, rows, cols, h, attn_out, gate_msa, block.norm2.weight, scale_mlp,
826
+ shift_mlp, idx, h2, q8, s, cols, h.stride(0), attn_out.stride(0), stride_tab, block.norm2.eps)
827
+ proj = _fp8_gemm(q8, s, block.ff.net[0].proj)
828
+ inner = proj.shape[1] // 2
829
+ f8 = torch.empty((rows, inner), device=device, dtype=FP8)
830
+ _row_launch(_swiglu_fp8_kernel, rows, inner, proj, f8, s, inner, proj.stride(0))
831
+ ff_out = _fp8_gemm(f8, s, block.ff.net[2])
832
+ out = torch.empty_like(h)
833
+ _row_launch(_res_gate_kernel, rows, cols, h2, ff_out, gate_mlp, idx, out, cols, h2.stride(0), ff_out.stride(0),
834
+ stride_tab)
835
+ return out.view(batch, rows, cols)
836
+
837
+
838
+ @torch.no_grad()
839
+ def fuse_dit_blocks(transformer) -> None:
840
+ """Fused forwards for every transformer block. Needs `install_adaln_tables` and `quantize_fp8` first (and q, k, v
841
+ as separate FP8Linear, which are concatenated into one QKV GEMM here)."""
842
+ import types
843
+
844
+ for block in transformer.transformer_blocks:
845
+ attn = block.attn
846
+ parts = [attn.to_q, attn.to_k, attn.to_v]
847
+ fused = FP8Linear.__new__(FP8Linear)
848
+ nn.Module.__init__(fused)
849
+ fused.register_buffer("weight", torch.cat([p.weight for p in parts]))
850
+ fused.register_buffer("weight_scale", torch.cat([p.weight_scale for p in parts], dim=1))
851
+ fused.bias, fused.out_dtype, fused.fast_accum = None, parts[0].out_dtype, parts[0].fast_accum
852
+ fused.in_features, fused.out_features = parts[0].in_features, sum(p.out_features for p in parts)
853
+ block._fused_qkv = fused
854
+ del attn.to_q, attn.to_k, attn.to_v
855
+ block.forward = types.MethodType(_fused_block_forward, block)
856
+ _free_cache()
857
+
requirements.txt ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ torch==2.13.0
2
+ torchvision==0.28.0
3
+ transformers==5.20.2
4
+ kernels==0.17.2
5
+ einops
6
+ accelerate
7
+ safetensors
8
+ huggingface_hub
9
+ pillow
10
+ numpy
11
+ diffusers @ git+https://github.com/huggingface/diffusers@c55ec27943462eeeb65c177d6ff7d8272d21d7a7