multimodalart HF Staff commited on
Commit
98e0470
·
verified ·
1 Parent(s): 20e6d56

AoTI kernels for the cached decode blocks and the VAE decoder; NCII guard in a sidecar subprocess

Browse files
Files changed (3) hide show
  1. app.py +21 -53
  2. ncii_guard.py +110 -0
  3. qwen21_aoti.py +148 -0
app.py CHANGED
@@ -2,7 +2,7 @@
2
 
3
  Pipeline for one request:
4
 
5
- 1. NCII guard (CPU, main process) — image-input requests have their prompt screened
6
  with ``hfmlsoc/ncii-guard-v02`` before any GPU time is reserved.
7
  2. Optional prompt enhancement — delegated over the Gradio API to a companion Space
8
  that keeps ``Qwen-Image-2.1-PE-T2I`` and ``Qwen-Image-2.1-PE-I2I`` warm.
@@ -27,12 +27,13 @@ import time # noqa: E402
27
  import gradio as gr # noqa: E402
28
  import torch # noqa: E402
29
  from diffusers import QwenImage21Pipeline # noqa: E402
30
- from huggingface_hub import snapshot_download # noqa: E402
31
  from PIL import Image # noqa: E402
32
- from transformers import AutoModelForSequenceClassification, AutoTokenizer # noqa: E402
 
33
 
34
  MODEL_ID = "Qwen/Qwen-Image-2.1"
35
- GUARD_ID = "hfmlsoc/ncii-guard-v02"
 
36
  PE_SPACE_ID = os.environ.get("PE_SPACE_ID", "hugging-apps/qwen-image-2-1-prompt-enhancer")
37
 
38
  MAX_SEED = 2**31 - 1
@@ -56,6 +57,7 @@ ASPECT_RATIOS = {
56
  "9:16": (1536, 2752),
57
  }
58
  RESOLUTION_CHOICES = [("1K — fastest", 1024), ("1.5K", 1536), ("2K — model default", 2048)]
 
59
 
60
  TRANSPARENCY_INFO = (
61
  "Describe what to generate, or how to edit the input images (refer to them as "
@@ -85,61 +87,25 @@ pipe.vae.enable_tiling(
85
  tile_sample_stride_height=1152,
86
  tile_sample_stride_width=1152,
87
  )
88
- print("[load] pipeline ready", flush=True)
89
 
90
- # The guard is a 270M classifier and stays on the CPU in the main process, so a prompt can
91
- # be rejected before any GPU allocation is requested. Its tokenizer carries the homoglyph /
92
- # zero-width normalizer, so it must come from this repo.
93
- def _guard_snapshot() -> str:
94
- """Fetch the guard and normalise its ``rope_parameters`` for transformers >= 5.16.
 
 
 
 
95
 
96
- The repo was saved with transformers 5.15, whose Gemma3 config mixes per-layer-type
97
- rope dicts with scalar siblings (``"rope_type": "default"``, ``"rope_theta": null``).
98
- Newer ``validate_rope`` walks every value expecting a mapping and trips on the scalars,
99
- so we mirror the snapshot into a directory that keeps only the per-layer-type entries.
100
- """
101
- src = snapshot_download(
102
- GUARD_ID,
103
- allow_patterns=["config.json", "tokenizer.json", "tokenizer_config.json", "model.safetensors"],
104
- )
105
- dst = os.path.join(tempfile.gettempdir(), "ncii-guard-v02-normalised")
106
- os.makedirs(dst, exist_ok=True)
107
- for name in os.listdir(src):
108
- link = os.path.join(dst, name)
109
- if not os.path.exists(link):
110
- os.symlink(os.path.realpath(os.path.join(src, name)), link)
111
-
112
- config = json.load(open(os.path.join(src, "config.json")))
113
- rope = config.get("rope_parameters")
114
- if isinstance(rope, dict):
115
- per_layer = {k: v for k, v in rope.items() if isinstance(v, dict)}
116
- if per_layer:
117
- config["rope_parameters"] = per_layer
118
- config_path = os.path.join(dst, "config.json")
119
- if os.path.islink(config_path):
120
- os.remove(config_path)
121
- with open(config_path, "w") as fh:
122
- json.dump(config, fh)
123
- return dst
124
-
125
-
126
- print(f"[load] {GUARD_ID} …", flush=True)
127
- _guard_dir = _guard_snapshot()
128
- guard_tokenizer = AutoTokenizer.from_pretrained(_guard_dir)
129
- guard_model = AutoModelForSequenceClassification.from_pretrained(
130
- _guard_dir, dtype=torch.float32
131
- ).eval()
132
- print(f"[load] guard ready ({guard_model.config.id2label})", flush=True)
133
 
134
 
135
  def guard_score(text: str) -> float:
136
  """Probability that ``text`` is a request for non-consensual intimate imagery."""
137
- batch = guard_tokenizer(
138
- text or "", truncation=True, max_length=256, padding=True, return_tensors="pt"
139
- )
140
- with torch.no_grad():
141
- logits = guard_model(**batch).logits.float()
142
- return torch.softmax(logits, dim=-1)[0, 1].item()
143
 
144
 
145
  def check_guard(prompt: str) -> None:
@@ -263,6 +229,8 @@ def estimate_duration(
263
  n_images = len(input_images or [])
264
  tokens = (res / 16.0) ** 2
265
  per_step = 2.05e-5 * tokens**1.21 * (2.0 if cfg > 1.0 else 1.0)
 
 
266
  fixed = 3.0 + 2.0 * n_images + 1.5e-6 * res * res
267
  return int(min(340, math.ceil((fixed + steps * per_step) * 1.12)))
268
 
 
2
 
3
  Pipeline for one request:
4
 
5
+ 1. NCII guard (CPU, sidecar subprocess) — image-input requests have their prompt screened
6
  with ``hfmlsoc/ncii-guard-v02`` before any GPU time is reserved.
7
  2. Optional prompt enhancement — delegated over the Gradio API to a companion Space
8
  that keeps ``Qwen-Image-2.1-PE-T2I`` and ``Qwen-Image-2.1-PE-I2I`` warm.
 
27
  import gradio as gr # noqa: E402
28
  import torch # noqa: E402
29
  from diffusers import QwenImage21Pipeline # noqa: E402
 
30
  from PIL import Image # noqa: E402
31
+ import ncii_guard # noqa: E402
32
+ from qwen21_aoti import aoti_load_pipeline # noqa: E402
33
 
34
  MODEL_ID = "Qwen/Qwen-Image-2.1"
35
+ AOTI_REPO = os.environ.get("QWEN21_AOTI_REPO", "hugging-apps/qwen-image-2-1-aoti")
36
+ GUARD_ID = ncii_guard.GUARD_ID
37
  PE_SPACE_ID = os.environ.get("PE_SPACE_ID", "hugging-apps/qwen-image-2-1-prompt-enhancer")
38
 
39
  MAX_SEED = 2**31 - 1
 
57
  "9:16": (1536, 2752),
58
  }
59
  RESOLUTION_CHOICES = [("1K — fastest", 1024), ("1.5K", 1536), ("2K — model default", 2048)]
60
+ AOTI_STEP_FACTOR = 0.78
61
 
62
  TRANSPARENCY_INFO = (
63
  "Describe what to generate, or how to edit the input images (refer to them as "
 
87
  tile_sample_stride_height=1152,
88
  tile_sample_stride_width=1152,
89
  )
 
90
 
91
+ AOTI_LOADED = []
92
+ if os.environ.get("QWEN21_AOTI", "1") != "0":
93
+ try:
94
+ AOTI_LOADED, _aoti_config = aoti_load_pipeline(pipe, AOTI_REPO, token=os.environ.get("HF_TOKEN"))
95
+ print(f"[load] aoti kernels from {AOTI_REPO}: {AOTI_LOADED} (torch {_aoti_config['torch']})", flush=True)
96
+ except Exception as exc: # noqa: BLE001
97
+ print(f"[load] aoti unavailable, running eager: {exc!r}", flush=True)
98
+ AOTI_BLOCKS = "QwenImage21DecodeBlock" in AOTI_LOADED
99
+ print("[load] pipeline ready", flush=True)
100
 
101
+ print(f"[load] {GUARD_ID} sidecar …", flush=True)
102
+ ncii_guard.start()
103
+ print("[load] guard ready", flush=True)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
104
 
105
 
106
  def guard_score(text: str) -> float:
107
  """Probability that ``text`` is a request for non-consensual intimate imagery."""
108
+ return ncii_guard.score(text or "")
 
 
 
 
 
109
 
110
 
111
  def check_guard(prompt: str) -> None:
 
229
  n_images = len(input_images or [])
230
  tokens = (res / 16.0) ** 2
231
  per_step = 2.05e-5 * tokens**1.21 * (2.0 if cfg > 1.0 else 1.0)
232
+ if AOTI_BLOCKS:
233
+ per_step *= AOTI_STEP_FACTOR
234
  fixed = 3.0 + 2.0 * n_images + 1.5e-6 * res * res
235
  return int(min(340, math.ceil((fixed + steps * per_step) * 1.12)))
236
 
ncii_guard.py ADDED
@@ -0,0 +1,110 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import json
4
+ import os
5
+ import select
6
+ import subprocess
7
+ import sys
8
+ import tempfile
9
+ import threading
10
+
11
+ GUARD_ID = "hfmlsoc/ncii-guard-v02"
12
+
13
+ _lock = threading.Lock()
14
+ _process: subprocess.Popen | None = None
15
+
16
+
17
+ def _read(timeout: float) -> dict:
18
+ readable, _, _ = select.select([_process.stdout], [], [], timeout)
19
+ if not readable:
20
+ raise TimeoutError(f"the guard did not answer within {timeout}s")
21
+ line = _process.stdout.readline()
22
+ if not line:
23
+ raise EOFError("the guard process died")
24
+ return json.loads(line)
25
+
26
+
27
+ def _spawn() -> None:
28
+ global _process
29
+ _process = subprocess.Popen(
30
+ [sys.executable, os.path.abspath(__file__)],
31
+ stdin=subprocess.PIPE,
32
+ stdout=subprocess.PIPE,
33
+ text=True,
34
+ bufsize=1,
35
+ )
36
+ ready = _read(600.0)
37
+ if ready.get("status") != "ready":
38
+ raise RuntimeError(f"guard failed to start: {ready}")
39
+
40
+
41
+ def start() -> None:
42
+ with _lock:
43
+ _spawn()
44
+
45
+
46
+ def score(prompt: str, timeout: float = 60.0) -> float:
47
+ with _lock:
48
+ for attempt in (0, 1):
49
+ try:
50
+ if _process is None or _process.poll() is not None:
51
+ _spawn()
52
+ _process.stdin.write(json.dumps({"prompt": prompt or ""}) + "\n")
53
+ _process.stdin.flush()
54
+ return float(_read(timeout)["score"])
55
+ except Exception:
56
+ if attempt:
57
+ raise
58
+ if _process is not None and _process.poll() is None:
59
+ _process.kill()
60
+
61
+
62
+ def _snapshot() -> str:
63
+ from huggingface_hub import snapshot_download
64
+
65
+ src = snapshot_download(
66
+ GUARD_ID,
67
+ allow_patterns=["config.json", "tokenizer.json", "tokenizer_config.json", "model.safetensors"],
68
+ )
69
+ dst = os.path.join(tempfile.gettempdir(), "ncii-guard-v02-normalised")
70
+ os.makedirs(dst, exist_ok=True)
71
+ for name in os.listdir(src):
72
+ link = os.path.join(dst, name)
73
+ if not os.path.exists(link):
74
+ os.symlink(os.path.realpath(os.path.join(src, name)), link)
75
+
76
+ config = json.load(open(os.path.join(src, "config.json")))
77
+ rope = config.get("rope_parameters")
78
+ if isinstance(rope, dict):
79
+ per_layer = {k: v for k, v in rope.items() if isinstance(v, dict)}
80
+ if per_layer:
81
+ config["rope_parameters"] = per_layer
82
+ config_path = os.path.join(dst, "config.json")
83
+ if os.path.islink(config_path):
84
+ os.remove(config_path)
85
+ with open(config_path, "w") as fh:
86
+ json.dump(config, fh)
87
+ return dst
88
+
89
+
90
+ def _serve() -> None:
91
+ protocol = os.fdopen(os.dup(1), "w", buffering=1)
92
+ os.dup2(2, 1)
93
+
94
+ import torch
95
+ from transformers import AutoModelForSequenceClassification, AutoTokenizer
96
+
97
+ guard_dir = _snapshot()
98
+ tokenizer = AutoTokenizer.from_pretrained(guard_dir)
99
+ model = AutoModelForSequenceClassification.from_pretrained(guard_dir, dtype=torch.float32).eval()
100
+ protocol.write(json.dumps({"status": "ready", "id2label": model.config.id2label}) + "\n")
101
+ for line in sys.stdin:
102
+ text = json.loads(line)["prompt"]
103
+ batch = tokenizer(text, truncation=True, max_length=256, padding=True, return_tensors="pt")
104
+ with torch.no_grad():
105
+ logits = model(**batch).logits.float()
106
+ protocol.write(json.dumps({"score": torch.softmax(logits, dim=-1)[0, 1].item()}) + "\n")
107
+
108
+
109
+ if __name__ == "__main__":
110
+ _serve()
qwen21_aoti.py ADDED
@@ -0,0 +1,148 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json
2
+ from pathlib import Path
3
+
4
+ import torch
5
+ import torch.nn as nn
6
+ import torch.nn.functional as F
7
+ from huggingface_hub import hf_hub_download
8
+
9
+ BLOCK_NAME = "QwenImage21DecodeBlock"
10
+ DECODER_NAME = "QwenImage21Decoder"
11
+ PACKAGE_FILENAME = "package.pt2"
12
+ INDUCTOR_CONFIGS = {"max_autotune": True, "coordinate_descent_tuning": True, "triton.cudagraphs": False}
13
+
14
+
15
+ def rotate(x, rotary_emb):
16
+ xf = x.float().unflatten(-1, (-1, 2))
17
+ xr, xi = xf[..., 0], xf[..., 1]
18
+ cos = rotary_emb[None, :, None, :, 0]
19
+ sin = rotary_emb[None, :, None, :, 1]
20
+ return torch.stack([xr * cos - xi * sin, xr * sin + xi * cos], dim=-1).flatten(3).type_as(x)
21
+
22
+
23
+ class QwenImage21DecodeBlock(nn.Module):
24
+ def __init__(self, block):
25
+ super().__init__()
26
+ self.block = block
27
+
28
+ def forward(self, hidden_states, modulation, rotary_emb, cached_key, cached_value):
29
+ block = self.block
30
+ attn = block.attn
31
+ mod1, mod2 = modulation[:-1].unsqueeze(1).chunk(2, dim=-1)
32
+ scale1, gate1 = mod1.chunk(2, dim=-1)
33
+ scale2, gate2 = mod2.chunk(2, dim=-1)
34
+
35
+ normed = block.img_norm1(hidden_states) * (1 + scale1)
36
+ query = attn.to_q(normed).unflatten(-1, (attn.heads, -1))
37
+ key = attn.to_k(normed).unflatten(-1, (attn.heads, -1))
38
+ value = attn.to_v(normed).unflatten(-1, (attn.heads, -1))
39
+ query = rotate(attn.norm_q(query).to(value.dtype), rotary_emb)
40
+ key = rotate(attn.norm_k(key).to(value.dtype), rotary_emb)
41
+ key = torch.cat([cached_key, key], dim=1)
42
+ value = torch.cat([cached_value, value], dim=1)
43
+ out = F.scaled_dot_product_attention(
44
+ query.permute(0, 2, 1, 3), key.permute(0, 2, 1, 3), value.permute(0, 2, 1, 3)
45
+ )
46
+ out = attn.to_out[0](out.permute(0, 2, 1, 3).flatten(2, 3).type_as(query))
47
+ hidden_states = hidden_states + gate1.tanh() * out
48
+
49
+ normed = block.img_norm2(hidden_states) * (1 + scale2)
50
+ return hidden_states + gate2.tanh() * block.img_mlp(normed)
51
+
52
+
53
+ class QwenImage21Decoder(nn.Module):
54
+ def __init__(self, decoder, num_convs):
55
+ super().__init__()
56
+ self.decoder = decoder
57
+ self.num_convs = num_convs
58
+
59
+ def forward(self, x):
60
+ return self.decoder(x, feat_cache=[None] * self.num_convs, feat_idx=[0], first_chunk=True)
61
+
62
+
63
+ def block_dynamic_shapes():
64
+ auto = torch.export.Dim.AUTO
65
+ return ({1: auto}, None, {0: auto}, {1: auto}, {1: auto})
66
+
67
+
68
+ def decoder_dynamic_shapes():
69
+ auto = torch.export.Dim.AUTO
70
+ return ({3: auto, 4: auto},)
71
+
72
+
73
+ def _decode_block_forward(compiled, eager):
74
+ def forward(
75
+ hidden_states,
76
+ modulation,
77
+ rotary_emb=None,
78
+ attention_mask=None,
79
+ target_token_mask=None,
80
+ layer_cache=None,
81
+ kv_cache_mode=None,
82
+ cache_write_slice=None,
83
+ segments=None,
84
+ key_valid=None,
85
+ ):
86
+ if (
87
+ kv_cache_mode == "cached"
88
+ and attention_mask is None
89
+ and layer_cache is not None
90
+ and hidden_states.shape[0] == 1
91
+ ):
92
+ cached_key, cached_value = layer_cache.get()
93
+ return compiled(
94
+ hidden_states.contiguous(),
95
+ modulation.contiguous(),
96
+ torch.view_as_real(rotary_emb).contiguous(),
97
+ cached_key,
98
+ cached_value,
99
+ )
100
+ return eager(
101
+ hidden_states,
102
+ modulation,
103
+ rotary_emb=rotary_emb,
104
+ attention_mask=attention_mask,
105
+ target_token_mask=target_token_mask,
106
+ layer_cache=layer_cache,
107
+ kv_cache_mode=kv_cache_mode,
108
+ cache_write_slice=cache_write_slice,
109
+ segments=segments,
110
+ key_valid=key_valid,
111
+ )
112
+
113
+ return forward
114
+
115
+
116
+ def _decoder_forward(compiled, eager):
117
+ def forward(x, feat_cache=None, feat_idx=None, first_chunk=False):
118
+ if first_chunk and x.shape[0] == 1 and x.shape[2] == 1:
119
+ return compiled(x.contiguous())
120
+ return eager(x, feat_cache=feat_cache, feat_idx=feat_idx, first_chunk=first_chunk)
121
+
122
+ return forward
123
+
124
+
125
+ def aoti_load_pipeline(pipe, repo_id, token=None, blocks=True, decoder=True):
126
+ from spaces.zero.torch.aoti import LazyAOTIModel, aoti_patch
127
+
128
+ config = json.loads(Path(hf_hub_download(repo_id, "config.json", token=token)).read_text())
129
+ if config.get("torch") != torch.__version__:
130
+ raise RuntimeError(f"artifacts built for torch {config.get('torch')}, runtime has {torch.__version__}")
131
+ loaded = []
132
+ if blocks and BLOCK_NAME in config.get("artifacts", {}):
133
+ block_model = LazyAOTIModel(hf_hub_download(repo_id, PACKAGE_FILENAME, subfolder=BLOCK_NAME, token=token))
134
+ for block in pipe.transformer.transformer_blocks:
135
+ wrapper = QwenImage21DecodeBlock(block)
136
+ aoti_patch(wrapper, block_model)
137
+ block.forward = _decode_block_forward(wrapper, block.forward)
138
+ loaded.append(BLOCK_NAME)
139
+ if decoder and DECODER_NAME in config.get("artifacts", {}):
140
+ decoder_model = LazyAOTIModel(
141
+ hf_hub_download(repo_id, PACKAGE_FILENAME, subfolder=DECODER_NAME, token=token)
142
+ )
143
+ vae = pipe.vae
144
+ wrapper = QwenImage21Decoder(vae.decoder, vae._cached_conv_counts["decoder"])
145
+ aoti_patch(wrapper, decoder_model)
146
+ vae.decoder.forward = _decoder_forward(wrapper, vae.decoder.forward)
147
+ loaded.append(DECODER_NAME)
148
+ return loaded, config