Onise commited on
Commit
015983b
·
verified ·
1 Parent(s): 4dd73bb

Update qwenimage/qwen_fa3_processor.py

Browse files
Files changed (1) hide show
  1. qwenimage/qwen_fa3_processor.py +98 -85
qwenimage/qwen_fa3_processor.py CHANGED
@@ -1,117 +1,136 @@
1
  """
2
- Paired with a good language model. Thanks!
3
  """
4
 
5
  import torch
 
 
6
  from typing import Optional, Tuple
7
  from diffusers.models.transformers.transformer_qwenimage import apply_rotary_emb_qwen
8
 
9
- import torch.nn.functional as F
10
-
11
  _flash_attn_func = None
12
- _kernels_err = None
13
-
14
- try:
15
- from kernels import get_kernel
16
- # Blackwell (sm_120+) is not yet supported by the vllm-flash-attn3 kernel binary
17
- _cap = torch.cuda.get_device_capability() if torch.cuda.is_available() else (0, 0)
18
- if _cap >= (12, 0):
19
- _kernels_err = RuntimeError(
20
- f"GPU compute capability sm_{_cap[0]}{_cap[1]} (Blackwell+) is not supported "
21
- "by the kernels-community/vllm-flash-attn3 binary; using SDPA fallback."
22
- )
23
- else:
 
 
 
 
 
 
 
 
 
 
24
  _k = get_kernel("kernels-community/vllm-flash-attn3")
25
  _flash_attn_func = _k.flash_attn_func
26
- except Exception as e:
27
- _flash_attn_func = None
28
- _kernels_err = e
 
 
 
 
29
 
30
 
31
- def _fa3_available() -> bool:
32
- return _flash_attn_func is not None
 
 
 
 
33
 
 
 
 
 
 
 
34
 
35
- def _sdpa_fallback(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor) -> torch.Tensor:
36
- # q/k/v: (B, S, H, D_h) → SDPA expects (B, H, S, D_h)
37
- q = q.transpose(1, 2)
38
- k = k.transpose(1, 2)
39
- v = v.transpose(1, 2)
40
- out = F.scaled_dot_product_attention(q, k, v, is_causal=False)
41
- return out.transpose(1, 2) # back to (B, S, H, D_h)
42
 
43
 
44
- @torch.library.custom_op("flash::flash_attn_func", mutates_args=())
45
- def flash_attn_func(
46
- q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, causal: bool = False
47
- ) -> torch.Tensor:
48
- outputs, lse = _flash_attn_func(q, k, v, causal=causal)
49
- return outputs
50
 
51
 
52
- @flash_attn_func.register_fake
53
- def _(q, k, v, **kwargs):
54
- return torch.empty_like(q).contiguous()
 
 
 
 
 
 
 
55
 
56
 
 
 
 
57
  class QwenDoubleStreamAttnProcessorFA3:
58
- """
59
- FA3-based attention processor for Qwen double-stream architecture.
60
- Computes joint attention over concatenated [text, image] streams using vLLM FlashAttention-3
61
- accessed via Hugging Face `kernels`.
62
-
63
- Notes / limitations:
64
- - General attention masks are not supported here (FA3 path). `is_causal=False` and no arbitrary mask.
65
- - Optional windowed attention / sink tokens / softcap can be plumbed through if you use those features.
66
- - Expects an available `apply_rotary_emb_qwen` in scope (same as your non-FA3 processor).
67
- """
68
 
69
- _attention_backend = "fa3" # for parity with your other processors, not used internally
 
70
 
71
- def __init__(self):
72
- if not _fa3_available():
73
- print(f"[QwenDoubleStreamAttnProcessorFA3] FA3 unavailable, using SDPA fallback. Reason: {_kernels_err}")
 
74
 
75
  @torch.no_grad()
76
  def __call__(
77
  self,
78
- attn, # Attention module with to_q/to_k/to_v/add_*_proj, norms, to_out, to_add_out, and .heads
79
- hidden_states: torch.FloatTensor, # (B, S_img, D_model) image stream
80
- encoder_hidden_states: torch.FloatTensor = None, # (B, S_txt, D_model) text stream
81
- encoder_hidden_states_mask: torch.FloatTensor = None, # unused in FA3 path
82
- attention_mask: Optional[torch.FloatTensor] = None, # unused in FA3 path
83
- image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, # (img_freqs, txt_freqs)
84
  ) -> Tuple[torch.FloatTensor, torch.FloatTensor]:
85
  if encoder_hidden_states is None:
86
- raise ValueError("QwenDoubleStreamAttnProcessorFA3 requires encoder_hidden_states (text stream).")
87
- if attention_mask is not None:
88
- # FA3 kernel path here does not consume arbitrary masks; fail fast to avoid silent correctness issues.
89
- raise NotImplementedError("attention_mask is not supported in this FA3 implementation.")
90
 
91
  B, S_img, _ = hidden_states.shape
92
  S_txt = encoder_hidden_states.shape[1]
93
 
94
- # ---- QKV projections (image/sample stream) ----
95
- img_q = attn.to_q(hidden_states) # (B, S_img, D)
96
  img_k = attn.to_k(hidden_states)
97
  img_v = attn.to_v(hidden_states)
98
-
99
- # ---- QKV projections (text/context stream) ----
100
- txt_q = attn.add_q_proj(encoder_hidden_states) # (B, S_txt, D)
101
  txt_k = attn.add_k_proj(encoder_hidden_states)
102
  txt_v = attn.add_v_proj(encoder_hidden_states)
103
 
104
- # ---- Reshape to (B, S, H, D_h) ----
105
  H = attn.heads
106
  img_q = img_q.unflatten(-1, (H, -1))
107
  img_k = img_k.unflatten(-1, (H, -1))
108
  img_v = img_v.unflatten(-1, (H, -1))
109
-
110
  txt_q = txt_q.unflatten(-1, (H, -1))
111
  txt_k = txt_k.unflatten(-1, (H, -1))
112
  txt_v = txt_v.unflatten(-1, (H, -1))
113
 
114
- # ---- Q/K normalization (per your module contract) ----
115
  if getattr(attn, "norm_q", None) is not None:
116
  img_q = attn.norm_q(img_q)
117
  if getattr(attn, "norm_k", None) is not None:
@@ -121,38 +140,32 @@ class QwenDoubleStreamAttnProcessorFA3:
121
  if getattr(attn, "norm_added_k", None) is not None:
122
  txt_k = attn.norm_added_k(txt_k)
123
 
124
- # ---- RoPE (Qwen variant) ----
125
  if image_rotary_emb is not None:
126
  img_freqs, txt_freqs = image_rotary_emb
127
- # expects tensors shaped (B, S, H, D_h)
128
  img_q = apply_rotary_emb_qwen(img_q, img_freqs, use_real=False)
129
  img_k = apply_rotary_emb_qwen(img_k, img_freqs, use_real=False)
130
  txt_q = apply_rotary_emb_qwen(txt_q, txt_freqs, use_real=False)
131
  txt_k = apply_rotary_emb_qwen(txt_k, txt_freqs, use_real=False)
132
 
133
- # ---- Joint attention over [text, image] along sequence axis ----
134
- # Shapes: (B, S_total, H, D_h)
135
  q = torch.cat([txt_q, img_q], dim=1)
136
  k = torch.cat([txt_k, img_k], dim=1)
137
  v = torch.cat([txt_v, img_v], dim=1)
138
 
139
- if _fa3_available():
140
- out = flash_attn_func(q, k, v, causal=False) # out: (B, S_total, H, D_h)
141
- else:
142
- out = _sdpa_fallback(q, k, v) # out: (B, S_total, H, D_h)
143
 
144
- # ---- Back to (B, S, D_model) ----
145
  out = out.flatten(2, 3).to(q.dtype)
146
 
147
- # Split back to text / image segments
148
- txt_attn_out = out[:, :S_txt, :]
149
- img_attn_out = out[:, S_txt:, :]
150
 
151
- # ---- Output projections ----
152
- img_attn_out = attn.to_out[0](img_attn_out)
153
  if len(attn.to_out) > 1:
154
- img_attn_out = attn.to_out[1](img_attn_out) # dropout if present
155
 
156
- txt_attn_out = attn.to_add_out(txt_attn_out)
157
 
158
- return img_attn_out, txt_attn_out
 
1
  """
2
+ QwenDoubleStreamAttnProcessorFA3 - Optimized lazy loading + Blackwell support
3
  """
4
 
5
  import torch
6
+ import torch.nn.functional as F
7
+ import warnings
8
  from typing import Optional, Tuple
9
  from diffusers.models.transformers.transformer_qwenimage import apply_rotary_emb_qwen
10
 
11
+ # Global lazy cache
12
+ _fa3_available: bool = False
13
  _flash_attn_func = None
14
+ _fa3_unavailable_reason: str = ""
15
+
16
+
17
+ def _ensure_fa3_loaded():
18
+ """Lazy-load FA3 kernel only the first time it's needed."""
19
+ global _fa3_available, _flash_attn_func, _fa3_unavailable_reason
20
+
21
+ if _fa3_available or _fa3_unavailable_reason:
22
+ return
23
+
24
+ # Blackwell (sm_100+) not supported yet by vllm-flash-attn3
25
+ if torch.cuda.is_available():
26
+ cap = torch.cuda.get_device_capability()
27
+ if cap[0] >= 10:
28
+ _fa3_unavailable_reason = (
29
+ "FlashAttention-3 is not yet supported on Blackwell (sm_100) GPUs. "
30
+ "Falling back to scaled-dot-product attention (SDPA)."
31
+ )
32
+ return
33
+
34
+ try:
35
+ from kernels import get_kernel
36
  _k = get_kernel("kernels-community/vllm-flash-attn3")
37
  _flash_attn_func = _k.flash_attn_func
38
+ _fa3_available = True
39
+ except Exception as e:
40
+ _fa3_unavailable_reason = (
41
+ f"FlashAttention-3 via Hugging Face kernels unavailable.\n"
42
+ f"get_kernel('kernels-community/vllm-flash-attn3') failed with:\n{e}\n"
43
+ "Falling back to SDPA."
44
+ )
45
 
46
 
47
+ # ---------------------------------------------------------------------------
48
+ # Custom op (only registered once FA3 is confirmed available)
49
+ # ---------------------------------------------------------------------------
50
+ def _register_fa3_op():
51
+ if not _fa3_available:
52
+ return
53
 
54
+ @torch.library.custom_op("flash::flash_attn_func", mutates_args=())
55
+ def flash_attn_func(
56
+ q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, causal: bool = False
57
+ ) -> torch.Tensor:
58
+ output, _ = _flash_attn_func(q, k, v, causal=causal)
59
+ return output
60
 
61
+ @flash_attn_func.register_fake
62
+ def _fake(q, k, v, causal=False):
63
+ return torch.empty_like(q).contiguous()
 
 
 
 
64
 
65
 
66
+ _register_fa3_op() # safe to call at import — does nothing if FA3 unavailable
 
 
 
 
 
67
 
68
 
69
+ # ---------------------------------------------------------------------------
70
+ # SDPA fallback
71
+ # ---------------------------------------------------------------------------
72
+ def _sdpa_attention(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor) -> torch.Tensor:
73
+ # (B, S, H, D_h) → SDPA format
74
+ q = q.transpose(1, 2)
75
+ k = k.transpose(1, 2)
76
+ v = v.transpose(1, 2)
77
+ out = F.scaled_dot_product_attention(q, k, v, is_causal=False)
78
+ return out.transpose(1, 2)
79
 
80
 
81
+ # ---------------------------------------------------------------------------
82
+ # Main processor
83
+ # ---------------------------------------------------------------------------
84
  class QwenDoubleStreamAttnProcessorFA3:
85
+ def __init__(self):
86
+ _ensure_fa3_loaded()
87
+ self._backend = "fa3" if _fa3_available else "sdpa"
 
 
 
 
 
 
 
88
 
89
+ if not _fa3_available:
90
+ warnings.warn(f"QwenDoubleStreamAttnProcessorFA3: {_fa3_unavailable_reason}", stacklevel=2)
91
 
92
+ def _attend(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor) -> torch.Tensor:
93
+ if self._backend == "fa3":
94
+ return torch.ops.flash.flash_attn_func(q, k, v, False) # custom op
95
+ return _sdpa_attention(q, k, v)
96
 
97
  @torch.no_grad()
98
  def __call__(
99
  self,
100
+ attn,
101
+ hidden_states: torch.FloatTensor,
102
+ encoder_hidden_states: torch.FloatTensor = None,
103
+ encoder_hidden_states_mask: torch.FloatTensor = None,
104
+ attention_mask: Optional[torch.FloatTensor] = None,
105
+ image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
106
  ) -> Tuple[torch.FloatTensor, torch.FloatTensor]:
107
  if encoder_hidden_states is None:
108
+ raise ValueError("QwenDoubleStreamAttnProcessorFA3 requires encoder_hidden_states.")
109
+
110
+ if attention_mask is not None and self._backend == "fa3":
111
+ raise NotImplementedError("attention_mask not supported on FA3 path.")
112
 
113
  B, S_img, _ = hidden_states.shape
114
  S_txt = encoder_hidden_states.shape[1]
115
 
116
+ # QKV projections
117
+ img_q = attn.to_q(hidden_states)
118
  img_k = attn.to_k(hidden_states)
119
  img_v = attn.to_v(hidden_states)
120
+ txt_q = attn.add_q_proj(encoder_hidden_states)
 
 
121
  txt_k = attn.add_k_proj(encoder_hidden_states)
122
  txt_v = attn.add_v_proj(encoder_hidden_states)
123
 
124
+ # Reshape
125
  H = attn.heads
126
  img_q = img_q.unflatten(-1, (H, -1))
127
  img_k = img_k.unflatten(-1, (H, -1))
128
  img_v = img_v.unflatten(-1, (H, -1))
 
129
  txt_q = txt_q.unflatten(-1, (H, -1))
130
  txt_k = txt_k.unflatten(-1, (H, -1))
131
  txt_v = txt_v.unflatten(-1, (H, -1))
132
 
133
+ # Norms
134
  if getattr(attn, "norm_q", None) is not None:
135
  img_q = attn.norm_q(img_q)
136
  if getattr(attn, "norm_k", None) is not None:
 
140
  if getattr(attn, "norm_added_k", None) is not None:
141
  txt_k = attn.norm_added_k(txt_k)
142
 
143
+ # RoPE
144
  if image_rotary_emb is not None:
145
  img_freqs, txt_freqs = image_rotary_emb
 
146
  img_q = apply_rotary_emb_qwen(img_q, img_freqs, use_real=False)
147
  img_k = apply_rotary_emb_qwen(img_k, img_freqs, use_real=False)
148
  txt_q = apply_rotary_emb_qwen(txt_q, txt_freqs, use_real=False)
149
  txt_k = apply_rotary_emb_qwen(txt_k, txt_freqs, use_real=False)
150
 
151
+ # Joint attention
 
152
  q = torch.cat([txt_q, img_q], dim=1)
153
  k = torch.cat([txt_k, img_k], dim=1)
154
  v = torch.cat([txt_v, img_v], dim=1)
155
 
156
+ out = self._attend(q, k, v)
 
 
 
157
 
158
+ # Back to (B, S, D)
159
  out = out.flatten(2, 3).to(q.dtype)
160
 
161
+ # Split & output projections
162
+ txt_out = out[:, :S_txt, :]
163
+ img_out = out[:, S_txt:, :]
164
 
165
+ img_out = attn.to_out[0](img_out)
 
166
  if len(attn.to_out) > 1:
167
+ img_out = attn.to_out[1](img_out)
168
 
169
+ txt_out = attn.to_add_out(txt_out)
170
 
171
+ return img_out, txt_out