Spaces:
Running on Zero
Running on Zero
Update qwenimage/qwen_fa3_processor.py
Browse files- qwenimage/qwen_fa3_processor.py +98 -85
qwenimage/qwen_fa3_processor.py
CHANGED
|
@@ -1,117 +1,136 @@
|
|
| 1 |
"""
|
| 2 |
-
|
| 3 |
"""
|
| 4 |
|
| 5 |
import torch
|
|
|
|
|
|
|
| 6 |
from typing import Optional, Tuple
|
| 7 |
from diffusers.models.transformers.transformer_qwenimage import apply_rotary_emb_qwen
|
| 8 |
|
| 9 |
-
|
| 10 |
-
|
| 11 |
_flash_attn_func = None
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 24 |
_k = get_kernel("kernels-community/vllm-flash-attn3")
|
| 25 |
_flash_attn_func = _k.flash_attn_func
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 29 |
|
| 30 |
|
| 31 |
-
|
| 32 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 33 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 34 |
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
k = k.transpose(1, 2)
|
| 39 |
-
v = v.transpose(1, 2)
|
| 40 |
-
out = F.scaled_dot_product_attention(q, k, v, is_causal=False)
|
| 41 |
-
return out.transpose(1, 2) # back to (B, S, H, D_h)
|
| 42 |
|
| 43 |
|
| 44 |
-
|
| 45 |
-
def flash_attn_func(
|
| 46 |
-
q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, causal: bool = False
|
| 47 |
-
) -> torch.Tensor:
|
| 48 |
-
outputs, lse = _flash_attn_func(q, k, v, causal=causal)
|
| 49 |
-
return outputs
|
| 50 |
|
| 51 |
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 55 |
|
| 56 |
|
|
|
|
|
|
|
|
|
|
| 57 |
class QwenDoubleStreamAttnProcessorFA3:
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
accessed via Hugging Face `kernels`.
|
| 62 |
-
|
| 63 |
-
Notes / limitations:
|
| 64 |
-
- General attention masks are not supported here (FA3 path). `is_causal=False` and no arbitrary mask.
|
| 65 |
-
- Optional windowed attention / sink tokens / softcap can be plumbed through if you use those features.
|
| 66 |
-
- Expects an available `apply_rotary_emb_qwen` in scope (same as your non-FA3 processor).
|
| 67 |
-
"""
|
| 68 |
|
| 69 |
-
|
|
|
|
| 70 |
|
| 71 |
-
def
|
| 72 |
-
if
|
| 73 |
-
|
|
|
|
| 74 |
|
| 75 |
@torch.no_grad()
|
| 76 |
def __call__(
|
| 77 |
self,
|
| 78 |
-
attn,
|
| 79 |
-
hidden_states: torch.FloatTensor,
|
| 80 |
-
encoder_hidden_states: torch.FloatTensor = None,
|
| 81 |
-
encoder_hidden_states_mask: torch.FloatTensor = None,
|
| 82 |
-
attention_mask: Optional[torch.FloatTensor] = None,
|
| 83 |
-
image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
|
| 84 |
) -> Tuple[torch.FloatTensor, torch.FloatTensor]:
|
| 85 |
if encoder_hidden_states is None:
|
| 86 |
-
raise ValueError("QwenDoubleStreamAttnProcessorFA3 requires encoder_hidden_states
|
| 87 |
-
|
| 88 |
-
|
| 89 |
-
raise NotImplementedError("attention_mask
|
| 90 |
|
| 91 |
B, S_img, _ = hidden_states.shape
|
| 92 |
S_txt = encoder_hidden_states.shape[1]
|
| 93 |
|
| 94 |
-
#
|
| 95 |
-
img_q = attn.to_q(hidden_states)
|
| 96 |
img_k = attn.to_k(hidden_states)
|
| 97 |
img_v = attn.to_v(hidden_states)
|
| 98 |
-
|
| 99 |
-
# ---- QKV projections (text/context stream) ----
|
| 100 |
-
txt_q = attn.add_q_proj(encoder_hidden_states) # (B, S_txt, D)
|
| 101 |
txt_k = attn.add_k_proj(encoder_hidden_states)
|
| 102 |
txt_v = attn.add_v_proj(encoder_hidden_states)
|
| 103 |
|
| 104 |
-
#
|
| 105 |
H = attn.heads
|
| 106 |
img_q = img_q.unflatten(-1, (H, -1))
|
| 107 |
img_k = img_k.unflatten(-1, (H, -1))
|
| 108 |
img_v = img_v.unflatten(-1, (H, -1))
|
| 109 |
-
|
| 110 |
txt_q = txt_q.unflatten(-1, (H, -1))
|
| 111 |
txt_k = txt_k.unflatten(-1, (H, -1))
|
| 112 |
txt_v = txt_v.unflatten(-1, (H, -1))
|
| 113 |
|
| 114 |
-
#
|
| 115 |
if getattr(attn, "norm_q", None) is not None:
|
| 116 |
img_q = attn.norm_q(img_q)
|
| 117 |
if getattr(attn, "norm_k", None) is not None:
|
|
@@ -121,38 +140,32 @@ class QwenDoubleStreamAttnProcessorFA3:
|
|
| 121 |
if getattr(attn, "norm_added_k", None) is not None:
|
| 122 |
txt_k = attn.norm_added_k(txt_k)
|
| 123 |
|
| 124 |
-
#
|
| 125 |
if image_rotary_emb is not None:
|
| 126 |
img_freqs, txt_freqs = image_rotary_emb
|
| 127 |
-
# expects tensors shaped (B, S, H, D_h)
|
| 128 |
img_q = apply_rotary_emb_qwen(img_q, img_freqs, use_real=False)
|
| 129 |
img_k = apply_rotary_emb_qwen(img_k, img_freqs, use_real=False)
|
| 130 |
txt_q = apply_rotary_emb_qwen(txt_q, txt_freqs, use_real=False)
|
| 131 |
txt_k = apply_rotary_emb_qwen(txt_k, txt_freqs, use_real=False)
|
| 132 |
|
| 133 |
-
#
|
| 134 |
-
# Shapes: (B, S_total, H, D_h)
|
| 135 |
q = torch.cat([txt_q, img_q], dim=1)
|
| 136 |
k = torch.cat([txt_k, img_k], dim=1)
|
| 137 |
v = torch.cat([txt_v, img_v], dim=1)
|
| 138 |
|
| 139 |
-
|
| 140 |
-
out = flash_attn_func(q, k, v, causal=False) # out: (B, S_total, H, D_h)
|
| 141 |
-
else:
|
| 142 |
-
out = _sdpa_fallback(q, k, v) # out: (B, S_total, H, D_h)
|
| 143 |
|
| 144 |
-
#
|
| 145 |
out = out.flatten(2, 3).to(q.dtype)
|
| 146 |
|
| 147 |
-
# Split
|
| 148 |
-
|
| 149 |
-
|
| 150 |
|
| 151 |
-
|
| 152 |
-
img_attn_out = attn.to_out[0](img_attn_out)
|
| 153 |
if len(attn.to_out) > 1:
|
| 154 |
-
|
| 155 |
|
| 156 |
-
|
| 157 |
|
| 158 |
-
return
|
|
|
|
| 1 |
"""
|
| 2 |
+
QwenDoubleStreamAttnProcessorFA3 - Optimized lazy loading + Blackwell support
|
| 3 |
"""
|
| 4 |
|
| 5 |
import torch
|
| 6 |
+
import torch.nn.functional as F
|
| 7 |
+
import warnings
|
| 8 |
from typing import Optional, Tuple
|
| 9 |
from diffusers.models.transformers.transformer_qwenimage import apply_rotary_emb_qwen
|
| 10 |
|
| 11 |
+
# Global lazy cache
|
| 12 |
+
_fa3_available: bool = False
|
| 13 |
_flash_attn_func = None
|
| 14 |
+
_fa3_unavailable_reason: str = ""
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def _ensure_fa3_loaded():
|
| 18 |
+
"""Lazy-load FA3 kernel only the first time it's needed."""
|
| 19 |
+
global _fa3_available, _flash_attn_func, _fa3_unavailable_reason
|
| 20 |
+
|
| 21 |
+
if _fa3_available or _fa3_unavailable_reason:
|
| 22 |
+
return
|
| 23 |
+
|
| 24 |
+
# Blackwell (sm_100+) not supported yet by vllm-flash-attn3
|
| 25 |
+
if torch.cuda.is_available():
|
| 26 |
+
cap = torch.cuda.get_device_capability()
|
| 27 |
+
if cap[0] >= 10:
|
| 28 |
+
_fa3_unavailable_reason = (
|
| 29 |
+
"FlashAttention-3 is not yet supported on Blackwell (sm_100) GPUs. "
|
| 30 |
+
"Falling back to scaled-dot-product attention (SDPA)."
|
| 31 |
+
)
|
| 32 |
+
return
|
| 33 |
+
|
| 34 |
+
try:
|
| 35 |
+
from kernels import get_kernel
|
| 36 |
_k = get_kernel("kernels-community/vllm-flash-attn3")
|
| 37 |
_flash_attn_func = _k.flash_attn_func
|
| 38 |
+
_fa3_available = True
|
| 39 |
+
except Exception as e:
|
| 40 |
+
_fa3_unavailable_reason = (
|
| 41 |
+
f"FlashAttention-3 via Hugging Face kernels unavailable.\n"
|
| 42 |
+
f"get_kernel('kernels-community/vllm-flash-attn3') failed with:\n{e}\n"
|
| 43 |
+
"Falling back to SDPA."
|
| 44 |
+
)
|
| 45 |
|
| 46 |
|
| 47 |
+
# ---------------------------------------------------------------------------
|
| 48 |
+
# Custom op (only registered once FA3 is confirmed available)
|
| 49 |
+
# ---------------------------------------------------------------------------
|
| 50 |
+
def _register_fa3_op():
|
| 51 |
+
if not _fa3_available:
|
| 52 |
+
return
|
| 53 |
|
| 54 |
+
@torch.library.custom_op("flash::flash_attn_func", mutates_args=())
|
| 55 |
+
def flash_attn_func(
|
| 56 |
+
q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, causal: bool = False
|
| 57 |
+
) -> torch.Tensor:
|
| 58 |
+
output, _ = _flash_attn_func(q, k, v, causal=causal)
|
| 59 |
+
return output
|
| 60 |
|
| 61 |
+
@flash_attn_func.register_fake
|
| 62 |
+
def _fake(q, k, v, causal=False):
|
| 63 |
+
return torch.empty_like(q).contiguous()
|
|
|
|
|
|
|
|
|
|
|
|
|
| 64 |
|
| 65 |
|
| 66 |
+
_register_fa3_op() # safe to call at import — does nothing if FA3 unavailable
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 67 |
|
| 68 |
|
| 69 |
+
# ---------------------------------------------------------------------------
|
| 70 |
+
# SDPA fallback
|
| 71 |
+
# ---------------------------------------------------------------------------
|
| 72 |
+
def _sdpa_attention(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor) -> torch.Tensor:
|
| 73 |
+
# (B, S, H, D_h) → SDPA format
|
| 74 |
+
q = q.transpose(1, 2)
|
| 75 |
+
k = k.transpose(1, 2)
|
| 76 |
+
v = v.transpose(1, 2)
|
| 77 |
+
out = F.scaled_dot_product_attention(q, k, v, is_causal=False)
|
| 78 |
+
return out.transpose(1, 2)
|
| 79 |
|
| 80 |
|
| 81 |
+
# ---------------------------------------------------------------------------
|
| 82 |
+
# Main processor
|
| 83 |
+
# ---------------------------------------------------------------------------
|
| 84 |
class QwenDoubleStreamAttnProcessorFA3:
|
| 85 |
+
def __init__(self):
|
| 86 |
+
_ensure_fa3_loaded()
|
| 87 |
+
self._backend = "fa3" if _fa3_available else "sdpa"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 88 |
|
| 89 |
+
if not _fa3_available:
|
| 90 |
+
warnings.warn(f"QwenDoubleStreamAttnProcessorFA3: {_fa3_unavailable_reason}", stacklevel=2)
|
| 91 |
|
| 92 |
+
def _attend(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor) -> torch.Tensor:
|
| 93 |
+
if self._backend == "fa3":
|
| 94 |
+
return torch.ops.flash.flash_attn_func(q, k, v, False) # custom op
|
| 95 |
+
return _sdpa_attention(q, k, v)
|
| 96 |
|
| 97 |
@torch.no_grad()
|
| 98 |
def __call__(
|
| 99 |
self,
|
| 100 |
+
attn,
|
| 101 |
+
hidden_states: torch.FloatTensor,
|
| 102 |
+
encoder_hidden_states: torch.FloatTensor = None,
|
| 103 |
+
encoder_hidden_states_mask: torch.FloatTensor = None,
|
| 104 |
+
attention_mask: Optional[torch.FloatTensor] = None,
|
| 105 |
+
image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
|
| 106 |
) -> Tuple[torch.FloatTensor, torch.FloatTensor]:
|
| 107 |
if encoder_hidden_states is None:
|
| 108 |
+
raise ValueError("QwenDoubleStreamAttnProcessorFA3 requires encoder_hidden_states.")
|
| 109 |
+
|
| 110 |
+
if attention_mask is not None and self._backend == "fa3":
|
| 111 |
+
raise NotImplementedError("attention_mask not supported on FA3 path.")
|
| 112 |
|
| 113 |
B, S_img, _ = hidden_states.shape
|
| 114 |
S_txt = encoder_hidden_states.shape[1]
|
| 115 |
|
| 116 |
+
# QKV projections
|
| 117 |
+
img_q = attn.to_q(hidden_states)
|
| 118 |
img_k = attn.to_k(hidden_states)
|
| 119 |
img_v = attn.to_v(hidden_states)
|
| 120 |
+
txt_q = attn.add_q_proj(encoder_hidden_states)
|
|
|
|
|
|
|
| 121 |
txt_k = attn.add_k_proj(encoder_hidden_states)
|
| 122 |
txt_v = attn.add_v_proj(encoder_hidden_states)
|
| 123 |
|
| 124 |
+
# Reshape
|
| 125 |
H = attn.heads
|
| 126 |
img_q = img_q.unflatten(-1, (H, -1))
|
| 127 |
img_k = img_k.unflatten(-1, (H, -1))
|
| 128 |
img_v = img_v.unflatten(-1, (H, -1))
|
|
|
|
| 129 |
txt_q = txt_q.unflatten(-1, (H, -1))
|
| 130 |
txt_k = txt_k.unflatten(-1, (H, -1))
|
| 131 |
txt_v = txt_v.unflatten(-1, (H, -1))
|
| 132 |
|
| 133 |
+
# Norms
|
| 134 |
if getattr(attn, "norm_q", None) is not None:
|
| 135 |
img_q = attn.norm_q(img_q)
|
| 136 |
if getattr(attn, "norm_k", None) is not None:
|
|
|
|
| 140 |
if getattr(attn, "norm_added_k", None) is not None:
|
| 141 |
txt_k = attn.norm_added_k(txt_k)
|
| 142 |
|
| 143 |
+
# RoPE
|
| 144 |
if image_rotary_emb is not None:
|
| 145 |
img_freqs, txt_freqs = image_rotary_emb
|
|
|
|
| 146 |
img_q = apply_rotary_emb_qwen(img_q, img_freqs, use_real=False)
|
| 147 |
img_k = apply_rotary_emb_qwen(img_k, img_freqs, use_real=False)
|
| 148 |
txt_q = apply_rotary_emb_qwen(txt_q, txt_freqs, use_real=False)
|
| 149 |
txt_k = apply_rotary_emb_qwen(txt_k, txt_freqs, use_real=False)
|
| 150 |
|
| 151 |
+
# Joint attention
|
|
|
|
| 152 |
q = torch.cat([txt_q, img_q], dim=1)
|
| 153 |
k = torch.cat([txt_k, img_k], dim=1)
|
| 154 |
v = torch.cat([txt_v, img_v], dim=1)
|
| 155 |
|
| 156 |
+
out = self._attend(q, k, v)
|
|
|
|
|
|
|
|
|
|
| 157 |
|
| 158 |
+
# Back to (B, S, D)
|
| 159 |
out = out.flatten(2, 3).to(q.dtype)
|
| 160 |
|
| 161 |
+
# Split & output projections
|
| 162 |
+
txt_out = out[:, :S_txt, :]
|
| 163 |
+
img_out = out[:, S_txt:, :]
|
| 164 |
|
| 165 |
+
img_out = attn.to_out[0](img_out)
|
|
|
|
| 166 |
if len(attn.to_out) > 1:
|
| 167 |
+
img_out = attn.to_out[1](img_out)
|
| 168 |
|
| 169 |
+
txt_out = attn.to_add_out(txt_out)
|
| 170 |
|
| 171 |
+
return img_out, txt_out
|