AoTI kernels for the cached decode blocks and the VAE decoder; NCII guard in a sidecar subprocess
Browse files- app.py +21 -53
- ncii_guard.py +110 -0
- qwen21_aoti.py +148 -0
app.py
CHANGED
|
@@ -2,7 +2,7 @@
|
|
| 2 |
|
| 3 |
Pipeline for one request:
|
| 4 |
|
| 5 |
-
1. NCII guard (CPU,
|
| 6 |
with ``hfmlsoc/ncii-guard-v02`` before any GPU time is reserved.
|
| 7 |
2. Optional prompt enhancement — delegated over the Gradio API to a companion Space
|
| 8 |
that keeps ``Qwen-Image-2.1-PE-T2I`` and ``Qwen-Image-2.1-PE-I2I`` warm.
|
|
@@ -27,12 +27,13 @@ import time # noqa: E402
|
|
| 27 |
import gradio as gr # noqa: E402
|
| 28 |
import torch # noqa: E402
|
| 29 |
from diffusers import QwenImage21Pipeline # noqa: E402
|
| 30 |
-
from huggingface_hub import snapshot_download # noqa: E402
|
| 31 |
from PIL import Image # noqa: E402
|
| 32 |
-
|
|
|
|
| 33 |
|
| 34 |
MODEL_ID = "Qwen/Qwen-Image-2.1"
|
| 35 |
-
|
|
|
|
| 36 |
PE_SPACE_ID = os.environ.get("PE_SPACE_ID", "hugging-apps/qwen-image-2-1-prompt-enhancer")
|
| 37 |
|
| 38 |
MAX_SEED = 2**31 - 1
|
|
@@ -56,6 +57,7 @@ ASPECT_RATIOS = {
|
|
| 56 |
"9:16": (1536, 2752),
|
| 57 |
}
|
| 58 |
RESOLUTION_CHOICES = [("1K — fastest", 1024), ("1.5K", 1536), ("2K — model default", 2048)]
|
|
|
|
| 59 |
|
| 60 |
TRANSPARENCY_INFO = (
|
| 61 |
"Describe what to generate, or how to edit the input images (refer to them as "
|
|
@@ -85,61 +87,25 @@ pipe.vae.enable_tiling(
|
|
| 85 |
tile_sample_stride_height=1152,
|
| 86 |
tile_sample_stride_width=1152,
|
| 87 |
)
|
| 88 |
-
print("[load] pipeline ready", flush=True)
|
| 89 |
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 95 |
|
| 96 |
-
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
so we mirror the snapshot into a directory that keeps only the per-layer-type entries.
|
| 100 |
-
"""
|
| 101 |
-
src = snapshot_download(
|
| 102 |
-
GUARD_ID,
|
| 103 |
-
allow_patterns=["config.json", "tokenizer.json", "tokenizer_config.json", "model.safetensors"],
|
| 104 |
-
)
|
| 105 |
-
dst = os.path.join(tempfile.gettempdir(), "ncii-guard-v02-normalised")
|
| 106 |
-
os.makedirs(dst, exist_ok=True)
|
| 107 |
-
for name in os.listdir(src):
|
| 108 |
-
link = os.path.join(dst, name)
|
| 109 |
-
if not os.path.exists(link):
|
| 110 |
-
os.symlink(os.path.realpath(os.path.join(src, name)), link)
|
| 111 |
-
|
| 112 |
-
config = json.load(open(os.path.join(src, "config.json")))
|
| 113 |
-
rope = config.get("rope_parameters")
|
| 114 |
-
if isinstance(rope, dict):
|
| 115 |
-
per_layer = {k: v for k, v in rope.items() if isinstance(v, dict)}
|
| 116 |
-
if per_layer:
|
| 117 |
-
config["rope_parameters"] = per_layer
|
| 118 |
-
config_path = os.path.join(dst, "config.json")
|
| 119 |
-
if os.path.islink(config_path):
|
| 120 |
-
os.remove(config_path)
|
| 121 |
-
with open(config_path, "w") as fh:
|
| 122 |
-
json.dump(config, fh)
|
| 123 |
-
return dst
|
| 124 |
-
|
| 125 |
-
|
| 126 |
-
print(f"[load] {GUARD_ID} …", flush=True)
|
| 127 |
-
_guard_dir = _guard_snapshot()
|
| 128 |
-
guard_tokenizer = AutoTokenizer.from_pretrained(_guard_dir)
|
| 129 |
-
guard_model = AutoModelForSequenceClassification.from_pretrained(
|
| 130 |
-
_guard_dir, dtype=torch.float32
|
| 131 |
-
).eval()
|
| 132 |
-
print(f"[load] guard ready ({guard_model.config.id2label})", flush=True)
|
| 133 |
|
| 134 |
|
| 135 |
def guard_score(text: str) -> float:
|
| 136 |
"""Probability that ``text`` is a request for non-consensual intimate imagery."""
|
| 137 |
-
|
| 138 |
-
text or "", truncation=True, max_length=256, padding=True, return_tensors="pt"
|
| 139 |
-
)
|
| 140 |
-
with torch.no_grad():
|
| 141 |
-
logits = guard_model(**batch).logits.float()
|
| 142 |
-
return torch.softmax(logits, dim=-1)[0, 1].item()
|
| 143 |
|
| 144 |
|
| 145 |
def check_guard(prompt: str) -> None:
|
|
@@ -263,6 +229,8 @@ def estimate_duration(
|
|
| 263 |
n_images = len(input_images or [])
|
| 264 |
tokens = (res / 16.0) ** 2
|
| 265 |
per_step = 2.05e-5 * tokens**1.21 * (2.0 if cfg > 1.0 else 1.0)
|
|
|
|
|
|
|
| 266 |
fixed = 3.0 + 2.0 * n_images + 1.5e-6 * res * res
|
| 267 |
return int(min(340, math.ceil((fixed + steps * per_step) * 1.12)))
|
| 268 |
|
|
|
|
| 2 |
|
| 3 |
Pipeline for one request:
|
| 4 |
|
| 5 |
+
1. NCII guard (CPU, sidecar subprocess) — image-input requests have their prompt screened
|
| 6 |
with ``hfmlsoc/ncii-guard-v02`` before any GPU time is reserved.
|
| 7 |
2. Optional prompt enhancement — delegated over the Gradio API to a companion Space
|
| 8 |
that keeps ``Qwen-Image-2.1-PE-T2I`` and ``Qwen-Image-2.1-PE-I2I`` warm.
|
|
|
|
| 27 |
import gradio as gr # noqa: E402
|
| 28 |
import torch # noqa: E402
|
| 29 |
from diffusers import QwenImage21Pipeline # noqa: E402
|
|
|
|
| 30 |
from PIL import Image # noqa: E402
|
| 31 |
+
import ncii_guard # noqa: E402
|
| 32 |
+
from qwen21_aoti import aoti_load_pipeline # noqa: E402
|
| 33 |
|
| 34 |
MODEL_ID = "Qwen/Qwen-Image-2.1"
|
| 35 |
+
AOTI_REPO = os.environ.get("QWEN21_AOTI_REPO", "hugging-apps/qwen-image-2-1-aoti")
|
| 36 |
+
GUARD_ID = ncii_guard.GUARD_ID
|
| 37 |
PE_SPACE_ID = os.environ.get("PE_SPACE_ID", "hugging-apps/qwen-image-2-1-prompt-enhancer")
|
| 38 |
|
| 39 |
MAX_SEED = 2**31 - 1
|
|
|
|
| 57 |
"9:16": (1536, 2752),
|
| 58 |
}
|
| 59 |
RESOLUTION_CHOICES = [("1K — fastest", 1024), ("1.5K", 1536), ("2K — model default", 2048)]
|
| 60 |
+
AOTI_STEP_FACTOR = 0.78
|
| 61 |
|
| 62 |
TRANSPARENCY_INFO = (
|
| 63 |
"Describe what to generate, or how to edit the input images (refer to them as "
|
|
|
|
| 87 |
tile_sample_stride_height=1152,
|
| 88 |
tile_sample_stride_width=1152,
|
| 89 |
)
|
|
|
|
| 90 |
|
| 91 |
+
AOTI_LOADED = []
|
| 92 |
+
if os.environ.get("QWEN21_AOTI", "1") != "0":
|
| 93 |
+
try:
|
| 94 |
+
AOTI_LOADED, _aoti_config = aoti_load_pipeline(pipe, AOTI_REPO, token=os.environ.get("HF_TOKEN"))
|
| 95 |
+
print(f"[load] aoti kernels from {AOTI_REPO}: {AOTI_LOADED} (torch {_aoti_config['torch']})", flush=True)
|
| 96 |
+
except Exception as exc: # noqa: BLE001
|
| 97 |
+
print(f"[load] aoti unavailable, running eager: {exc!r}", flush=True)
|
| 98 |
+
AOTI_BLOCKS = "QwenImage21DecodeBlock" in AOTI_LOADED
|
| 99 |
+
print("[load] pipeline ready", flush=True)
|
| 100 |
|
| 101 |
+
print(f"[load] {GUARD_ID} sidecar …", flush=True)
|
| 102 |
+
ncii_guard.start()
|
| 103 |
+
print("[load] guard ready", flush=True)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 104 |
|
| 105 |
|
| 106 |
def guard_score(text: str) -> float:
|
| 107 |
"""Probability that ``text`` is a request for non-consensual intimate imagery."""
|
| 108 |
+
return ncii_guard.score(text or "")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 109 |
|
| 110 |
|
| 111 |
def check_guard(prompt: str) -> None:
|
|
|
|
| 229 |
n_images = len(input_images or [])
|
| 230 |
tokens = (res / 16.0) ** 2
|
| 231 |
per_step = 2.05e-5 * tokens**1.21 * (2.0 if cfg > 1.0 else 1.0)
|
| 232 |
+
if AOTI_BLOCKS:
|
| 233 |
+
per_step *= AOTI_STEP_FACTOR
|
| 234 |
fixed = 3.0 + 2.0 * n_images + 1.5e-6 * res * res
|
| 235 |
return int(min(340, math.ceil((fixed + steps * per_step) * 1.12)))
|
| 236 |
|
ncii_guard.py
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import json
|
| 4 |
+
import os
|
| 5 |
+
import select
|
| 6 |
+
import subprocess
|
| 7 |
+
import sys
|
| 8 |
+
import tempfile
|
| 9 |
+
import threading
|
| 10 |
+
|
| 11 |
+
GUARD_ID = "hfmlsoc/ncii-guard-v02"
|
| 12 |
+
|
| 13 |
+
_lock = threading.Lock()
|
| 14 |
+
_process: subprocess.Popen | None = None
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def _read(timeout: float) -> dict:
|
| 18 |
+
readable, _, _ = select.select([_process.stdout], [], [], timeout)
|
| 19 |
+
if not readable:
|
| 20 |
+
raise TimeoutError(f"the guard did not answer within {timeout}s")
|
| 21 |
+
line = _process.stdout.readline()
|
| 22 |
+
if not line:
|
| 23 |
+
raise EOFError("the guard process died")
|
| 24 |
+
return json.loads(line)
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def _spawn() -> None:
|
| 28 |
+
global _process
|
| 29 |
+
_process = subprocess.Popen(
|
| 30 |
+
[sys.executable, os.path.abspath(__file__)],
|
| 31 |
+
stdin=subprocess.PIPE,
|
| 32 |
+
stdout=subprocess.PIPE,
|
| 33 |
+
text=True,
|
| 34 |
+
bufsize=1,
|
| 35 |
+
)
|
| 36 |
+
ready = _read(600.0)
|
| 37 |
+
if ready.get("status") != "ready":
|
| 38 |
+
raise RuntimeError(f"guard failed to start: {ready}")
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def start() -> None:
|
| 42 |
+
with _lock:
|
| 43 |
+
_spawn()
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def score(prompt: str, timeout: float = 60.0) -> float:
|
| 47 |
+
with _lock:
|
| 48 |
+
for attempt in (0, 1):
|
| 49 |
+
try:
|
| 50 |
+
if _process is None or _process.poll() is not None:
|
| 51 |
+
_spawn()
|
| 52 |
+
_process.stdin.write(json.dumps({"prompt": prompt or ""}) + "\n")
|
| 53 |
+
_process.stdin.flush()
|
| 54 |
+
return float(_read(timeout)["score"])
|
| 55 |
+
except Exception:
|
| 56 |
+
if attempt:
|
| 57 |
+
raise
|
| 58 |
+
if _process is not None and _process.poll() is None:
|
| 59 |
+
_process.kill()
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def _snapshot() -> str:
|
| 63 |
+
from huggingface_hub import snapshot_download
|
| 64 |
+
|
| 65 |
+
src = snapshot_download(
|
| 66 |
+
GUARD_ID,
|
| 67 |
+
allow_patterns=["config.json", "tokenizer.json", "tokenizer_config.json", "model.safetensors"],
|
| 68 |
+
)
|
| 69 |
+
dst = os.path.join(tempfile.gettempdir(), "ncii-guard-v02-normalised")
|
| 70 |
+
os.makedirs(dst, exist_ok=True)
|
| 71 |
+
for name in os.listdir(src):
|
| 72 |
+
link = os.path.join(dst, name)
|
| 73 |
+
if not os.path.exists(link):
|
| 74 |
+
os.symlink(os.path.realpath(os.path.join(src, name)), link)
|
| 75 |
+
|
| 76 |
+
config = json.load(open(os.path.join(src, "config.json")))
|
| 77 |
+
rope = config.get("rope_parameters")
|
| 78 |
+
if isinstance(rope, dict):
|
| 79 |
+
per_layer = {k: v for k, v in rope.items() if isinstance(v, dict)}
|
| 80 |
+
if per_layer:
|
| 81 |
+
config["rope_parameters"] = per_layer
|
| 82 |
+
config_path = os.path.join(dst, "config.json")
|
| 83 |
+
if os.path.islink(config_path):
|
| 84 |
+
os.remove(config_path)
|
| 85 |
+
with open(config_path, "w") as fh:
|
| 86 |
+
json.dump(config, fh)
|
| 87 |
+
return dst
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
def _serve() -> None:
|
| 91 |
+
protocol = os.fdopen(os.dup(1), "w", buffering=1)
|
| 92 |
+
os.dup2(2, 1)
|
| 93 |
+
|
| 94 |
+
import torch
|
| 95 |
+
from transformers import AutoModelForSequenceClassification, AutoTokenizer
|
| 96 |
+
|
| 97 |
+
guard_dir = _snapshot()
|
| 98 |
+
tokenizer = AutoTokenizer.from_pretrained(guard_dir)
|
| 99 |
+
model = AutoModelForSequenceClassification.from_pretrained(guard_dir, dtype=torch.float32).eval()
|
| 100 |
+
protocol.write(json.dumps({"status": "ready", "id2label": model.config.id2label}) + "\n")
|
| 101 |
+
for line in sys.stdin:
|
| 102 |
+
text = json.loads(line)["prompt"]
|
| 103 |
+
batch = tokenizer(text, truncation=True, max_length=256, padding=True, return_tensors="pt")
|
| 104 |
+
with torch.no_grad():
|
| 105 |
+
logits = model(**batch).logits.float()
|
| 106 |
+
protocol.write(json.dumps({"score": torch.softmax(logits, dim=-1)[0, 1].item()}) + "\n")
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
if __name__ == "__main__":
|
| 110 |
+
_serve()
|
qwen21_aoti.py
ADDED
|
@@ -0,0 +1,148 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import json
|
| 2 |
+
from pathlib import Path
|
| 3 |
+
|
| 4 |
+
import torch
|
| 5 |
+
import torch.nn as nn
|
| 6 |
+
import torch.nn.functional as F
|
| 7 |
+
from huggingface_hub import hf_hub_download
|
| 8 |
+
|
| 9 |
+
BLOCK_NAME = "QwenImage21DecodeBlock"
|
| 10 |
+
DECODER_NAME = "QwenImage21Decoder"
|
| 11 |
+
PACKAGE_FILENAME = "package.pt2"
|
| 12 |
+
INDUCTOR_CONFIGS = {"max_autotune": True, "coordinate_descent_tuning": True, "triton.cudagraphs": False}
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def rotate(x, rotary_emb):
|
| 16 |
+
xf = x.float().unflatten(-1, (-1, 2))
|
| 17 |
+
xr, xi = xf[..., 0], xf[..., 1]
|
| 18 |
+
cos = rotary_emb[None, :, None, :, 0]
|
| 19 |
+
sin = rotary_emb[None, :, None, :, 1]
|
| 20 |
+
return torch.stack([xr * cos - xi * sin, xr * sin + xi * cos], dim=-1).flatten(3).type_as(x)
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
class QwenImage21DecodeBlock(nn.Module):
|
| 24 |
+
def __init__(self, block):
|
| 25 |
+
super().__init__()
|
| 26 |
+
self.block = block
|
| 27 |
+
|
| 28 |
+
def forward(self, hidden_states, modulation, rotary_emb, cached_key, cached_value):
|
| 29 |
+
block = self.block
|
| 30 |
+
attn = block.attn
|
| 31 |
+
mod1, mod2 = modulation[:-1].unsqueeze(1).chunk(2, dim=-1)
|
| 32 |
+
scale1, gate1 = mod1.chunk(2, dim=-1)
|
| 33 |
+
scale2, gate2 = mod2.chunk(2, dim=-1)
|
| 34 |
+
|
| 35 |
+
normed = block.img_norm1(hidden_states) * (1 + scale1)
|
| 36 |
+
query = attn.to_q(normed).unflatten(-1, (attn.heads, -1))
|
| 37 |
+
key = attn.to_k(normed).unflatten(-1, (attn.heads, -1))
|
| 38 |
+
value = attn.to_v(normed).unflatten(-1, (attn.heads, -1))
|
| 39 |
+
query = rotate(attn.norm_q(query).to(value.dtype), rotary_emb)
|
| 40 |
+
key = rotate(attn.norm_k(key).to(value.dtype), rotary_emb)
|
| 41 |
+
key = torch.cat([cached_key, key], dim=1)
|
| 42 |
+
value = torch.cat([cached_value, value], dim=1)
|
| 43 |
+
out = F.scaled_dot_product_attention(
|
| 44 |
+
query.permute(0, 2, 1, 3), key.permute(0, 2, 1, 3), value.permute(0, 2, 1, 3)
|
| 45 |
+
)
|
| 46 |
+
out = attn.to_out[0](out.permute(0, 2, 1, 3).flatten(2, 3).type_as(query))
|
| 47 |
+
hidden_states = hidden_states + gate1.tanh() * out
|
| 48 |
+
|
| 49 |
+
normed = block.img_norm2(hidden_states) * (1 + scale2)
|
| 50 |
+
return hidden_states + gate2.tanh() * block.img_mlp(normed)
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
class QwenImage21Decoder(nn.Module):
|
| 54 |
+
def __init__(self, decoder, num_convs):
|
| 55 |
+
super().__init__()
|
| 56 |
+
self.decoder = decoder
|
| 57 |
+
self.num_convs = num_convs
|
| 58 |
+
|
| 59 |
+
def forward(self, x):
|
| 60 |
+
return self.decoder(x, feat_cache=[None] * self.num_convs, feat_idx=[0], first_chunk=True)
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def block_dynamic_shapes():
|
| 64 |
+
auto = torch.export.Dim.AUTO
|
| 65 |
+
return ({1: auto}, None, {0: auto}, {1: auto}, {1: auto})
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
def decoder_dynamic_shapes():
|
| 69 |
+
auto = torch.export.Dim.AUTO
|
| 70 |
+
return ({3: auto, 4: auto},)
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def _decode_block_forward(compiled, eager):
|
| 74 |
+
def forward(
|
| 75 |
+
hidden_states,
|
| 76 |
+
modulation,
|
| 77 |
+
rotary_emb=None,
|
| 78 |
+
attention_mask=None,
|
| 79 |
+
target_token_mask=None,
|
| 80 |
+
layer_cache=None,
|
| 81 |
+
kv_cache_mode=None,
|
| 82 |
+
cache_write_slice=None,
|
| 83 |
+
segments=None,
|
| 84 |
+
key_valid=None,
|
| 85 |
+
):
|
| 86 |
+
if (
|
| 87 |
+
kv_cache_mode == "cached"
|
| 88 |
+
and attention_mask is None
|
| 89 |
+
and layer_cache is not None
|
| 90 |
+
and hidden_states.shape[0] == 1
|
| 91 |
+
):
|
| 92 |
+
cached_key, cached_value = layer_cache.get()
|
| 93 |
+
return compiled(
|
| 94 |
+
hidden_states.contiguous(),
|
| 95 |
+
modulation.contiguous(),
|
| 96 |
+
torch.view_as_real(rotary_emb).contiguous(),
|
| 97 |
+
cached_key,
|
| 98 |
+
cached_value,
|
| 99 |
+
)
|
| 100 |
+
return eager(
|
| 101 |
+
hidden_states,
|
| 102 |
+
modulation,
|
| 103 |
+
rotary_emb=rotary_emb,
|
| 104 |
+
attention_mask=attention_mask,
|
| 105 |
+
target_token_mask=target_token_mask,
|
| 106 |
+
layer_cache=layer_cache,
|
| 107 |
+
kv_cache_mode=kv_cache_mode,
|
| 108 |
+
cache_write_slice=cache_write_slice,
|
| 109 |
+
segments=segments,
|
| 110 |
+
key_valid=key_valid,
|
| 111 |
+
)
|
| 112 |
+
|
| 113 |
+
return forward
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
def _decoder_forward(compiled, eager):
|
| 117 |
+
def forward(x, feat_cache=None, feat_idx=None, first_chunk=False):
|
| 118 |
+
if first_chunk and x.shape[0] == 1 and x.shape[2] == 1:
|
| 119 |
+
return compiled(x.contiguous())
|
| 120 |
+
return eager(x, feat_cache=feat_cache, feat_idx=feat_idx, first_chunk=first_chunk)
|
| 121 |
+
|
| 122 |
+
return forward
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
def aoti_load_pipeline(pipe, repo_id, token=None, blocks=True, decoder=True):
|
| 126 |
+
from spaces.zero.torch.aoti import LazyAOTIModel, aoti_patch
|
| 127 |
+
|
| 128 |
+
config = json.loads(Path(hf_hub_download(repo_id, "config.json", token=token)).read_text())
|
| 129 |
+
if config.get("torch") != torch.__version__:
|
| 130 |
+
raise RuntimeError(f"artifacts built for torch {config.get('torch')}, runtime has {torch.__version__}")
|
| 131 |
+
loaded = []
|
| 132 |
+
if blocks and BLOCK_NAME in config.get("artifacts", {}):
|
| 133 |
+
block_model = LazyAOTIModel(hf_hub_download(repo_id, PACKAGE_FILENAME, subfolder=BLOCK_NAME, token=token))
|
| 134 |
+
for block in pipe.transformer.transformer_blocks:
|
| 135 |
+
wrapper = QwenImage21DecodeBlock(block)
|
| 136 |
+
aoti_patch(wrapper, block_model)
|
| 137 |
+
block.forward = _decode_block_forward(wrapper, block.forward)
|
| 138 |
+
loaded.append(BLOCK_NAME)
|
| 139 |
+
if decoder and DECODER_NAME in config.get("artifacts", {}):
|
| 140 |
+
decoder_model = LazyAOTIModel(
|
| 141 |
+
hf_hub_download(repo_id, PACKAGE_FILENAME, subfolder=DECODER_NAME, token=token)
|
| 142 |
+
)
|
| 143 |
+
vae = pipe.vae
|
| 144 |
+
wrapper = QwenImage21Decoder(vae.decoder, vae._cached_conv_counts["decoder"])
|
| 145 |
+
aoti_patch(wrapper, decoder_model)
|
| 146 |
+
vae.decoder.forward = _decoder_forward(wrapper, vae.decoder.forward)
|
| 147 |
+
loaded.append(DECODER_NAME)
|
| 148 |
+
return loaded, config
|