liangsu9988 commited on
Commit
25af561
·
verified ·
1 Parent(s): fda325f

Uploaded using `kernel-builder`.

Browse files
Files changed (21) hide show
  1. benchmarks/benchmark.py +779 -0
  2. build/torch211-cxx11-cu128-x86_64-linux/__init__.py +487 -0
  3. build/torch211-cxx11-cu128-x86_64-linux/_flashrt_flex_attention_train_cuda_eaf7454.abi3.so +3 -0
  4. build/torch211-cxx11-cu128-x86_64-linux/_ops.py +9 -0
  5. build/torch211-cxx11-cu128-x86_64-linux/flashrt_flex_attention_train/__init__.py +26 -0
  6. build/torch211-cxx11-cu128-x86_64-linux/metadata.json +23 -0
  7. build/torch211-cxx11-cu130-x86_64-linux/__init__.py +487 -0
  8. build/torch211-cxx11-cu130-x86_64-linux/_flashrt_flex_attention_train_cuda_eaf7454.abi3.so +3 -0
  9. build/torch211-cxx11-cu130-x86_64-linux/_ops.py +9 -0
  10. build/torch211-cxx11-cu130-x86_64-linux/flashrt_flex_attention_train/__init__.py +26 -0
  11. build/torch211-cxx11-cu130-x86_64-linux/metadata.json +22 -0
  12. build/torch212-cxx11-cu130-x86_64-linux/__init__.py +487 -0
  13. build/torch212-cxx11-cu130-x86_64-linux/_flashrt_flex_attention_train_cuda_eaf7454.abi3.so +3 -0
  14. build/torch212-cxx11-cu130-x86_64-linux/_ops.py +9 -0
  15. build/torch212-cxx11-cu130-x86_64-linux/flashrt_flex_attention_train/__init__.py +26 -0
  16. build/torch212-cxx11-cu130-x86_64-linux/metadata.json +22 -0
  17. build/torch212-cxx11-cu132-x86_64-linux/__init__.py +487 -0
  18. build/torch212-cxx11-cu132-x86_64-linux/_flashrt_flex_attention_train_cuda_eaf7454.abi3.so +3 -0
  19. build/torch212-cxx11-cu132-x86_64-linux/_ops.py +9 -0
  20. build/torch212-cxx11-cu132-x86_64-linux/flashrt_flex_attention_train/__init__.py +26 -0
  21. build/torch212-cxx11-cu132-x86_64-linux/metadata.json +22 -0
benchmarks/benchmark.py ADDED
@@ -0,0 +1,779 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ from __future__ import annotations
3
+
4
+ import argparse
5
+ import json
6
+ import sys
7
+ import time
8
+ from pathlib import Path
9
+
10
+ import torch
11
+ import torch.nn.functional as F
12
+
13
+
14
+ ROOT = Path(__file__).resolve().parents[1]
15
+ sys.path.insert(0, str(ROOT / "torch-ext"))
16
+ import flashrt_flex_attention_train as flex_ops # noqa: E402
17
+
18
+
19
+ FLEX_TILE_PRESETS = {
20
+ "default": None,
21
+ # fwd fully autotuned; backward constrained to the consumer-GPU tiles
22
+ # (autotuned backward at GQA/D=256 exceeds 5090 shared memory).
23
+ "bwd_shrunk_only": {
24
+ "bwd_BLOCK_M1": 32,
25
+ "bwd_BLOCK_N1": 64,
26
+ "bwd_BLOCK_M2": 64,
27
+ "bwd_BLOCK_N2": 32,
28
+ },
29
+ "torch_default_explicit": {
30
+ "fwd_BLOCK_M": 32,
31
+ "fwd_BLOCK_N": 64,
32
+ "fwd_num_stages": 2,
33
+ "fwd_num_warps": 4,
34
+ "bwd_BLOCK_M1": 32,
35
+ "bwd_BLOCK_N1": 64,
36
+ "bwd_BLOCK_M2": 64,
37
+ "bwd_BLOCK_N2": 32,
38
+ "bwd_num_stages": 1,
39
+ "bwd_num_warps": 4,
40
+ },
41
+ "a100_d256_bwd_32x64": {
42
+ "fwd_BLOCK_M": 32,
43
+ "fwd_BLOCK_N": 64,
44
+ "fwd_num_stages": 2,
45
+ "fwd_num_warps": 4,
46
+ "bwd_BLOCK_M1": 32,
47
+ "bwd_BLOCK_N1": 64,
48
+ "bwd_BLOCK_M2": 64,
49
+ "bwd_BLOCK_N2": 32,
50
+ "bwd_num_stages": 3,
51
+ "bwd_num_warps": 4,
52
+ },
53
+ "a100_d256_bwd_32x128": {
54
+ "fwd_BLOCK_M": 32,
55
+ "fwd_BLOCK_N": 64,
56
+ "fwd_num_stages": 2,
57
+ "fwd_num_warps": 4,
58
+ "bwd_BLOCK_M1": 32,
59
+ "bwd_BLOCK_N1": 128,
60
+ "bwd_BLOCK_M2": 128,
61
+ "bwd_BLOCK_N2": 32,
62
+ "bwd_num_stages": 3,
63
+ "bwd_num_warps": 8,
64
+ },
65
+ "a100_d256_bwd_64x64": {
66
+ "fwd_BLOCK_M": 64,
67
+ "fwd_BLOCK_N": 64,
68
+ "fwd_num_stages": 3,
69
+ "fwd_num_warps": 4,
70
+ "bwd_BLOCK_M1": 64,
71
+ "bwd_BLOCK_N1": 64,
72
+ "bwd_BLOCK_M2": 64,
73
+ "bwd_BLOCK_N2": 64,
74
+ "bwd_num_stages": 3,
75
+ "bwd_num_warps": 4,
76
+ },
77
+ "a100_d256_bwd_64x128": {
78
+ "fwd_BLOCK_M": 64,
79
+ "fwd_BLOCK_N": 128,
80
+ "fwd_num_stages": 3,
81
+ "fwd_num_warps": 4,
82
+ "bwd_BLOCK_M1": 64,
83
+ "bwd_BLOCK_N1": 128,
84
+ "bwd_BLOCK_M2": 128,
85
+ "bwd_BLOCK_N2": 64,
86
+ "bwd_num_stages": 3,
87
+ "bwd_num_warps": 8,
88
+ },
89
+ "a100_d256_bwd_write_dq_false": {
90
+ "fwd_BLOCK_M": 32,
91
+ "fwd_BLOCK_N": 64,
92
+ "fwd_num_stages": 2,
93
+ "fwd_num_warps": 4,
94
+ "bwd_BLOCK_M1": 32,
95
+ "bwd_BLOCK_N1": 64,
96
+ "bwd_BLOCK_M2": 64,
97
+ "bwd_BLOCK_N2": 32,
98
+ "bwd_num_stages": 3,
99
+ "bwd_num_warps": 4,
100
+ "WRITE_DQ": False,
101
+ },
102
+ "a100_d256_prescale_safe": {
103
+ "fwd_BLOCK_M": 32,
104
+ "fwd_BLOCK_N": 64,
105
+ "fwd_num_stages": 2,
106
+ "fwd_num_warps": 4,
107
+ "bwd_BLOCK_M1": 32,
108
+ "bwd_BLOCK_N1": 64,
109
+ "bwd_BLOCK_M2": 64,
110
+ "bwd_BLOCK_N2": 32,
111
+ "bwd_num_stages": 3,
112
+ "bwd_num_warps": 4,
113
+ "PRESCALE_QK": True,
114
+ "ROWS_GUARANTEED_SAFE": True,
115
+ },
116
+ "a100_d256_contig_safe": {
117
+ "fwd_BLOCK_M": 32,
118
+ "fwd_BLOCK_N": 64,
119
+ "fwd_num_stages": 2,
120
+ "fwd_num_warps": 4,
121
+ "bwd_BLOCK_M1": 32,
122
+ "bwd_BLOCK_N1": 64,
123
+ "bwd_BLOCK_M2": 64,
124
+ "bwd_BLOCK_N2": 32,
125
+ "bwd_num_stages": 3,
126
+ "bwd_num_warps": 4,
127
+ "ROWS_GUARANTEED_SAFE": True,
128
+ "BLOCKS_ARE_CONTIGUOUS": True,
129
+ },
130
+ "a100_d256_contig_prescale": {
131
+ "fwd_BLOCK_M": 32,
132
+ "fwd_BLOCK_N": 64,
133
+ "fwd_num_stages": 2,
134
+ "fwd_num_warps": 4,
135
+ "bwd_BLOCK_M1": 32,
136
+ "bwd_BLOCK_N1": 64,
137
+ "bwd_BLOCK_M2": 64,
138
+ "bwd_BLOCK_N2": 32,
139
+ "bwd_num_stages": 3,
140
+ "bwd_num_warps": 4,
141
+ "PRESCALE_QK": True,
142
+ "ROWS_GUARANTEED_SAFE": True,
143
+ "BLOCKS_ARE_CONTIGUOUS": True,
144
+ },
145
+ "a100_d256_contig_write_dq_false": {
146
+ "fwd_BLOCK_M": 32,
147
+ "fwd_BLOCK_N": 64,
148
+ "fwd_num_stages": 2,
149
+ "fwd_num_warps": 4,
150
+ "bwd_BLOCK_M1": 32,
151
+ "bwd_BLOCK_N1": 64,
152
+ "bwd_BLOCK_M2": 64,
153
+ "bwd_BLOCK_N2": 32,
154
+ "bwd_num_stages": 3,
155
+ "bwd_num_warps": 4,
156
+ "ROWS_GUARANTEED_SAFE": True,
157
+ "BLOCKS_ARE_CONTIGUOUS": True,
158
+ "WRITE_DQ": False,
159
+ },
160
+ }
161
+
162
+
163
+ def bench(fn, warmup: int, iters: int) -> float:
164
+ for _ in range(warmup):
165
+ fn()
166
+ if torch.cuda.is_available():
167
+ torch.cuda.synchronize()
168
+ times = []
169
+ for _ in range(iters):
170
+ if torch.cuda.is_available():
171
+ t0 = torch.cuda.Event(enable_timing=True)
172
+ t1 = torch.cuda.Event(enable_timing=True)
173
+ t0.record()
174
+ fn()
175
+ t1.record()
176
+ torch.cuda.synchronize()
177
+ times.append(t0.elapsed_time(t1))
178
+ else:
179
+ start = time.perf_counter()
180
+ fn()
181
+ times.append((time.perf_counter() - start) * 1000.0)
182
+ times.sort()
183
+ return times[len(times) // 2]
184
+
185
+
186
+ def peak_bytes(fn, device: str) -> int:
187
+ if not device.startswith("cuda"):
188
+ fn()
189
+ return 0
190
+ torch.cuda.reset_peak_memory_stats()
191
+ fn()
192
+ torch.cuda.synchronize()
193
+ return int(torch.cuda.max_memory_allocated())
194
+
195
+
196
+ def best_nested_speedup(nested: dict) -> dict | None:
197
+ best = None
198
+ for block_key, by_preset in nested.items():
199
+ for preset, speedup in by_preset.items():
200
+ if not isinstance(speedup, float):
201
+ continue
202
+ if best is None or speedup > best["speedup"]:
203
+ best = {"block_mask": block_key, "preset": preset, "speedup": speedup}
204
+ return best
205
+
206
+
207
+ def make_inputs(args):
208
+ dtype = torch.bfloat16 if args.dtype == "bf16" else torch.float32
209
+ action_len = args.action_blocks * args.action_block_size
210
+ total = args.prefix_len + action_len
211
+ torch.manual_seed(args.seed)
212
+ q = torch.randn(args.batch, args.heads, total, args.head_dim, device=args.device, dtype=dtype)
213
+ kv_heads = args.kv_heads or args.heads
214
+ k = torch.randn(args.batch, kv_heads, total, args.head_dim, device=args.device, dtype=dtype)
215
+ v = torch.randn_like(k)
216
+ prefix_valid = torch.ones(args.batch, args.prefix_len, device=args.device, dtype=torch.bool)
217
+ prefix_att = torch.zeros_like(prefix_valid)
218
+ prefix_att[:, args.prefix_len // 2 :] = True
219
+ if args.no_prefix_mask:
220
+ prefix_valid = None
221
+ prefix_att = None
222
+ return q, k, v, prefix_valid, prefix_att
223
+
224
+
225
+ def make_flex_bundle(args, prefix_valid, prefix_att, block_size: tuple[int, int]):
226
+ try:
227
+ from torch.nn.attention.flex_attention import create_block_mask, flex_attention
228
+ except Exception:
229
+ return None
230
+
231
+ compiled_mask = torch.compile(create_block_mask, dynamic=False)
232
+ batch = args.batch
233
+ prefix_len = args.prefix_len
234
+ action_len = args.action_blocks * args.action_block_size
235
+ total_len = prefix_len + action_len
236
+ chunk = args.action_block_size
237
+ pad = prefix_valid
238
+ cum = torch.cumsum(prefix_att.to(torch.long), dim=1) if prefix_att is not None else None
239
+
240
+ def prefix_rows(b, h, q_idx, kv_idx):
241
+ if cum is None:
242
+ return kv_idx < prefix_len
243
+ kv_p = kv_idx.clamp(max=prefix_len - 1)
244
+ ok = (cum[b, kv_p] <= cum[b, q_idx]) & pad[b, kv_p] & pad[b, q_idx]
245
+ return (kv_idx < prefix_len) & ok
246
+
247
+ def action_rows(b, h, q_idx, kv_idx):
248
+ if pad is None:
249
+ to_prefix = kv_idx < prefix_len
250
+ else:
251
+ kv_p = kv_idx.clamp(max=prefix_len - 1)
252
+ to_prefix = (kv_idx < prefix_len) & pad[b, kv_p]
253
+ same_block = (q_idx // chunk) == ((kv_idx - prefix_len) // chunk)
254
+ return to_prefix | ((kv_idx >= prefix_len) & same_block)
255
+
256
+ block_prefix = compiled_mask(
257
+ prefix_rows,
258
+ B=batch,
259
+ H=None,
260
+ Q_LEN=prefix_len,
261
+ KV_LEN=total_len,
262
+ device=torch.device(args.device),
263
+ BLOCK_SIZE=block_size,
264
+ )
265
+ block_action = compiled_mask(
266
+ action_rows,
267
+ B=batch,
268
+ H=None,
269
+ Q_LEN=action_len,
270
+ KV_LEN=total_len,
271
+ device=torch.device(args.device),
272
+ BLOCK_SIZE=block_size,
273
+ )
274
+ compiled_calls = {}
275
+ scale = args.head_dim**-0.5
276
+ gqa = (args.kv_heads or args.heads) != args.heads
277
+ for name, options in FLEX_TILE_PRESETS.items():
278
+ def prefix_call(q, k, v, block_mask, options=options):
279
+ return flex_attention(
280
+ q,
281
+ k,
282
+ v,
283
+ block_mask=block_mask,
284
+ scale=scale,
285
+ enable_gqa=gqa,
286
+ kernel_options=options,
287
+ )
288
+
289
+ def action_call(q, k, v, block_mask, options=options):
290
+ return flex_attention(
291
+ q,
292
+ k,
293
+ v,
294
+ block_mask=block_mask,
295
+ scale=scale,
296
+ enable_gqa=gqa,
297
+ kernel_options=options,
298
+ )
299
+
300
+ compiled_calls[name] = (
301
+ torch.compile(prefix_call, dynamic=False),
302
+ torch.compile(action_call, dynamic=False),
303
+ )
304
+ return compiled_calls, block_prefix, block_action
305
+
306
+
307
+ def _manual_part(qs, ks, vs, m, scale):
308
+ """Materialized-logits attention: cuBLAS GEMMs + fused masked softmax.
309
+
310
+ Exact SDPA semantics (fp32 softmax); grouped-query handled as a strided
311
+ batched GEMM over (kv_head, group*Sq) without materializing repeated K/V.
312
+ """
313
+ B, H, Sq, D = qs.shape
314
+ Hk = ks.shape[1]
315
+ if Hk != H:
316
+ g = H // Hk
317
+ q2 = qs.reshape(B, Hk, g * Sq, D)
318
+ logits = (q2 @ ks.transpose(-1, -2)).reshape(B, H, Sq, -1)
319
+ else:
320
+ logits = qs @ ks.transpose(-1, -2)
321
+ logits = logits * scale + m
322
+ p = logits.float().softmax(dim=-1).to(qs.dtype)
323
+ if Hk != H:
324
+ out = (p.reshape(B, Hk, g * Sq, -1) @ vs).reshape(B, H, Sq, D)
325
+ else:
326
+ out = p @ vs
327
+ return out
328
+
329
+
330
+ _manual_part_compiled = None
331
+
332
+
333
+ def get_manual_part():
334
+ global _manual_part_compiled
335
+ if _manual_part_compiled is None:
336
+ _manual_part_compiled = torch.compile(_manual_part, dynamic=False)
337
+ return _manual_part_compiled
338
+
339
+
340
+ def parse_args() -> argparse.Namespace:
341
+ parser = argparse.ArgumentParser()
342
+ parser.add_argument("--device", default="cuda")
343
+ parser.add_argument("--dtype", choices=["bf16", "fp32"], default="bf16")
344
+ parser.add_argument("--batch", type=int, default=4)
345
+ parser.add_argument("--heads", type=int, default=8)
346
+ parser.add_argument("--kv-heads", type=int, default=None, help="KV heads for GQA (default: same as --heads)")
347
+ parser.add_argument("--head-dim", type=int, default=256)
348
+ parser.add_argument("--prefix-len", type=int, default=700)
349
+ parser.add_argument("--action-blocks", type=int, default=5)
350
+ parser.add_argument("--action-block-size", type=int, default=50)
351
+ parser.add_argument("--warmup", type=int, default=10)
352
+ parser.add_argument("--iters", type=int, default=30)
353
+ parser.add_argument("--seed", type=int, default=0)
354
+ parser.add_argument("--mode", choices=["fwd", "fwdbwd", "all"], default="all")
355
+ parser.add_argument(
356
+ "--backend",
357
+ default="all",
358
+ help="comma-separated subset of {package, torch-flex, manual} or 'all'",
359
+ )
360
+ parser.add_argument("--flex-preset", choices=sorted(FLEX_TILE_PRESETS), default="a100_d256_bwd_32x64")
361
+ parser.add_argument("--sweep-flex-presets", action="store_true")
362
+ parser.add_argument("--block-mask-q", type=int, default=128)
363
+ parser.add_argument("--block-mask-kv", type=int, default=128)
364
+ parser.add_argument("--sweep-block-mask-sizes", action="store_true")
365
+ parser.add_argument("--output")
366
+ parser.add_argument("--require-gates", action="store_true")
367
+ parser.add_argument("--no-prefix-mask", action="store_true")
368
+ return parser.parse_args()
369
+
370
+
371
+ def main() -> None:
372
+ args = parse_args()
373
+ if args.device.startswith("cuda") and not torch.cuda.is_available():
374
+ raise SystemExit("CUDA requested but not available")
375
+ backends = (
376
+ {"package", "torch-flex", "manual"}
377
+ if args.backend == "all"
378
+ else {b.strip() for b in args.backend.split(",") if b.strip()}
379
+ )
380
+
381
+ q, k, v, prefix_valid, prefix_att = make_inputs(args)
382
+ action_len = args.action_blocks * args.action_block_size
383
+ scale = args.head_dim**-0.5
384
+ pm, am = flex_ops.build_block_sparse_bool_masks(
385
+ prefix_valid,
386
+ prefix_att,
387
+ batch=args.batch,
388
+ prefix_len=args.prefix_len,
389
+ action_len=action_len,
390
+ action_block_size=args.action_block_size,
391
+ device=q.device,
392
+ )
393
+ full = torch.cat([pm, am], dim=1)
394
+ add_mask = torch.where(
395
+ full[:, None],
396
+ torch.zeros((), device=q.device, dtype=q.dtype),
397
+ torch.full((), flex_ops.MASK_VALUE_F32, device=q.device, dtype=q.dtype),
398
+ )
399
+ block_sizes = [(args.block_mask_q, args.block_mask_kv)]
400
+ if args.sweep_block_mask_sizes:
401
+ block_sizes = [(q, kv) for q in (16, 32, 64, 128) for kv in (32, 64, 128)]
402
+ flex_bundles = {
403
+ f"{q}x{kv}": make_flex_bundle(args, prefix_valid, prefix_att, (q, kv))
404
+ for q, kv in block_sizes
405
+ } if "torch-flex" in backends else {}
406
+ flex_presets = sorted(FLEX_TILE_PRESETS) if args.sweep_flex_presets else [args.flex_preset]
407
+
408
+ gqa = (args.kv_heads or args.heads) != args.heads
409
+
410
+ def sdpa_fwd():
411
+ out_p = F.scaled_dot_product_attention(
412
+ q[:, :, : args.prefix_len],
413
+ k,
414
+ v,
415
+ attn_mask=add_mask[:, :, : args.prefix_len],
416
+ scale=scale,
417
+ enable_gqa=gqa,
418
+ )
419
+ kd = torch.cat([k[:, :, : args.prefix_len].detach(), k[:, :, args.prefix_len :]], dim=2)
420
+ vd = torch.cat([v[:, :, : args.prefix_len].detach(), v[:, :, args.prefix_len :]], dim=2)
421
+ out_a = F.scaled_dot_product_attention(
422
+ q[:, :, args.prefix_len :],
423
+ kd,
424
+ vd,
425
+ attn_mask=add_mask[:, :, args.prefix_len :],
426
+ scale=scale,
427
+ enable_gqa=gqa,
428
+ )
429
+ return torch.cat([out_p, out_a], dim=2)
430
+
431
+ manual_part = get_manual_part() if "manual" in backends else None
432
+
433
+ def manual_eager_fwd():
434
+ out_p = _manual_part(q[:, :, : args.prefix_len], k, v, add_mask[:, :, : args.prefix_len], scale)
435
+ kd = torch.cat([k[:, :, : args.prefix_len].detach(), k[:, :, args.prefix_len :]], dim=2)
436
+ vd = torch.cat([v[:, :, : args.prefix_len].detach(), v[:, :, args.prefix_len :]], dim=2)
437
+ out_a = _manual_part(
438
+ q[:, :, args.prefix_len :], kd, vd, add_mask[:, :, args.prefix_len :], scale
439
+ )
440
+ return torch.cat([out_p, out_a], dim=2)
441
+
442
+ def manual_eager_fwdbwd():
443
+ qq = q.detach().clone().requires_grad_(True)
444
+ kk = k.detach().clone().requires_grad_(True)
445
+ vv = v.detach().clone().requires_grad_(True)
446
+ out_p = _manual_part(
447
+ qq[:, :, : args.prefix_len], kk, vv, add_mask[:, :, : args.prefix_len], scale
448
+ )
449
+ kd = torch.cat([kk[:, :, : args.prefix_len].detach(), kk[:, :, args.prefix_len :]], dim=2)
450
+ vd = torch.cat([vv[:, :, : args.prefix_len].detach(), vv[:, :, args.prefix_len :]], dim=2)
451
+ out_a = _manual_part(
452
+ qq[:, :, args.prefix_len :], kd, vd, add_mask[:, :, args.prefix_len :], scale
453
+ )
454
+ torch.cat([out_p, out_a], dim=2).float().square().mean().backward()
455
+
456
+ def _repeat_kv(t):
457
+ # the model's current path: materialize K/V to the q-head count
458
+ return t.repeat_interleave(args.heads // t.shape[1], dim=1)
459
+
460
+ def sdpa_repeat_fwd():
461
+ kr, vr = _repeat_kv(k), _repeat_kv(v)
462
+ out_p = F.scaled_dot_product_attention(
463
+ q[:, :, : args.prefix_len], kr, vr, attn_mask=add_mask[:, :, : args.prefix_len], scale=scale
464
+ )
465
+ kd = torch.cat([kr[:, :, : args.prefix_len].detach(), kr[:, :, args.prefix_len :]], dim=2)
466
+ vd = torch.cat([vr[:, :, : args.prefix_len].detach(), vr[:, :, args.prefix_len :]], dim=2)
467
+ out_a = F.scaled_dot_product_attention(
468
+ q[:, :, args.prefix_len :], kd, vd, attn_mask=add_mask[:, :, args.prefix_len :], scale=scale
469
+ )
470
+ return torch.cat([out_p, out_a], dim=2)
471
+
472
+ def manual_fwd():
473
+ out_p = manual_part(q[:, :, : args.prefix_len], k, v, add_mask[:, :, : args.prefix_len], scale)
474
+ kd = torch.cat([k[:, :, : args.prefix_len].detach(), k[:, :, args.prefix_len :]], dim=2)
475
+ vd = torch.cat([v[:, :, : args.prefix_len].detach(), v[:, :, args.prefix_len :]], dim=2)
476
+ out_a = manual_part(
477
+ q[:, :, args.prefix_len :], kd, vd, add_mask[:, :, args.prefix_len :], scale
478
+ )
479
+ return torch.cat([out_p, out_a], dim=2)
480
+
481
+ def package_fwd():
482
+ return flex_ops.flex_attention(
483
+ q,
484
+ k,
485
+ v,
486
+ prefix_len=args.prefix_len,
487
+ action_block_size=args.action_block_size,
488
+ prefix_valid=prefix_valid,
489
+ prefix_att=prefix_att,
490
+ scale=scale,
491
+ )
492
+
493
+ def torch_flex_fwd(preset: str, block_key: str):
494
+ if not flex_bundles:
495
+ raise RuntimeError("PyTorch FlexAttention is unavailable")
496
+ compiled_calls, block_prefix, block_action = flex_bundles[block_key]
497
+ prefix_call, action_call = compiled_calls[preset]
498
+ out_p = prefix_call(
499
+ q[:, :, : args.prefix_len],
500
+ k,
501
+ v,
502
+ block_prefix,
503
+ )
504
+ kd = torch.cat([k[:, :, : args.prefix_len].detach(), k[:, :, args.prefix_len :]], dim=2)
505
+ vd = torch.cat([v[:, :, : args.prefix_len].detach(), v[:, :, args.prefix_len :]], dim=2)
506
+ out_a = action_call(
507
+ q[:, :, args.prefix_len :],
508
+ kd,
509
+ vd,
510
+ block_action,
511
+ )
512
+ return torch.cat([out_p, out_a], dim=2)
513
+
514
+ report = {
515
+ "gpu": torch.cuda.get_device_name() if args.device.startswith("cuda") else "cpu",
516
+ "torch": torch.__version__,
517
+ "shape": {
518
+ "B": args.batch,
519
+ "heads": args.heads,
520
+ "kv_heads": args.kv_heads or args.heads,
521
+ "head_dim": args.head_dim,
522
+ "prefix_len": args.prefix_len,
523
+ "action_len": action_len,
524
+ "action_block_size": args.action_block_size,
525
+ },
526
+ "backend": args.backend,
527
+ "flex_presets": flex_presets,
528
+ "block_mask_sizes": list(flex_bundles) if flex_bundles else [],
529
+ "timestamp": time.strftime("%Y-%m-%d %H:%M:%S"),
530
+ }
531
+ report["package_has_native_ops"] = False
532
+ report["package_native_supported"] = False
533
+
534
+ with torch.no_grad():
535
+ sdpa_out = sdpa_fwd().float()
536
+ if "package" in backends:
537
+ report["package_fwd_max_abs_diff"] = float((sdpa_out - package_fwd().float()).abs().max())
538
+ if "manual" in backends:
539
+ report["manual_fwd_max_abs_diff"] = float((sdpa_out - manual_fwd().float()).abs().max())
540
+ if flex_bundles:
541
+ report["torch_flex_fwd_max_abs_diff"] = {}
542
+ for block_key in flex_bundles:
543
+ report["torch_flex_fwd_max_abs_diff"][block_key] = {}
544
+ for block_key in flex_bundles:
545
+ for preset in flex_presets:
546
+ try:
547
+ report["torch_flex_fwd_max_abs_diff"][block_key][preset] = float(
548
+ (sdpa_out - torch_flex_fwd(preset, block_key).float()).abs().max()
549
+ )
550
+ except Exception as exc:
551
+ report["torch_flex_fwd_max_abs_diff"][block_key][preset] = f"ERROR: {type(exc).__name__}: {exc}"
552
+ if args.mode in {"fwd", "all"}:
553
+ report["sdpa_fwd_ms"] = bench(sdpa_fwd, args.warmup, args.iters)
554
+ report["sdpa_peak_bytes"] = peak_bytes(sdpa_fwd, args.device)
555
+ if "package" in backends:
556
+ report["package_fwd_ms"] = bench(package_fwd, args.warmup, args.iters)
557
+ report["package_fwd_speedup"] = report["sdpa_fwd_ms"] / report["package_fwd_ms"]
558
+ report["package_peak_bytes"] = peak_bytes(package_fwd, args.device)
559
+ if "manual" in backends:
560
+ report["manual_fwd_ms"] = bench(manual_fwd, args.warmup, args.iters)
561
+ report["manual_fwd_speedup"] = report["sdpa_fwd_ms"] / report["manual_fwd_ms"]
562
+ report["manual_peak_bytes"] = peak_bytes(manual_fwd, args.device)
563
+ if gqa:
564
+ report["sdpa_repeat_fwd_ms"] = bench(sdpa_repeat_fwd, args.warmup, args.iters)
565
+ if "manual" in backends:
566
+ report["manual_eager_fwd_ms"] = bench(manual_eager_fwd, args.warmup, args.iters)
567
+ if flex_bundles:
568
+ report["torch_flex_fwd_ms"] = {}
569
+ report["torch_flex_fwd_speedup"] = {}
570
+ for block_key in flex_bundles:
571
+ report["torch_flex_fwd_ms"][block_key] = {}
572
+ report["torch_flex_fwd_speedup"][block_key] = {}
573
+ for block_key in flex_bundles:
574
+ for preset in flex_presets:
575
+ try:
576
+ ms = bench(lambda preset=preset, block_key=block_key: torch_flex_fwd(preset, block_key), args.warmup, args.iters)
577
+ except Exception as exc:
578
+ report["torch_flex_fwd_ms"][block_key][preset] = f"ERROR: {type(exc).__name__}: {exc}"
579
+ else:
580
+ report["torch_flex_fwd_ms"][block_key][preset] = ms
581
+ report["torch_flex_fwd_speedup"][block_key][preset] = report["sdpa_fwd_ms"] / ms
582
+ report["best_torch_flex_fwd"] = best_nested_speedup(report["torch_flex_fwd_speedup"])
583
+
584
+ if args.mode in {"fwdbwd", "all"}:
585
+ def sdpa_fwdbwd():
586
+ qq = q.detach().clone().requires_grad_(True)
587
+ kk = k.detach().clone().requires_grad_(True)
588
+ vv = v.detach().clone().requires_grad_(True)
589
+ out_p = F.scaled_dot_product_attention(
590
+ qq[:, :, : args.prefix_len],
591
+ kk,
592
+ vv,
593
+ attn_mask=add_mask[:, :, : args.prefix_len],
594
+ scale=scale,
595
+ enable_gqa=gqa,
596
+ )
597
+ kd = torch.cat([kk[:, :, : args.prefix_len].detach(), kk[:, :, args.prefix_len :]], dim=2)
598
+ vd = torch.cat([vv[:, :, : args.prefix_len].detach(), vv[:, :, args.prefix_len :]], dim=2)
599
+ out_a = F.scaled_dot_product_attention(
600
+ qq[:, :, args.prefix_len :],
601
+ kd,
602
+ vd,
603
+ attn_mask=add_mask[:, :, args.prefix_len :],
604
+ scale=scale,
605
+ enable_gqa=gqa,
606
+ )
607
+ torch.cat([out_p, out_a], dim=2).float().square().mean().backward()
608
+
609
+ def manual_fwdbwd():
610
+ qq = q.detach().clone().requires_grad_(True)
611
+ kk = k.detach().clone().requires_grad_(True)
612
+ vv = v.detach().clone().requires_grad_(True)
613
+ out_p = manual_part(
614
+ qq[:, :, : args.prefix_len], kk, vv, add_mask[:, :, : args.prefix_len], scale
615
+ )
616
+ kd = torch.cat([kk[:, :, : args.prefix_len].detach(), kk[:, :, args.prefix_len :]], dim=2)
617
+ vd = torch.cat([vv[:, :, : args.prefix_len].detach(), vv[:, :, args.prefix_len :]], dim=2)
618
+ out_a = manual_part(
619
+ qq[:, :, args.prefix_len :], kd, vd, add_mask[:, :, args.prefix_len :], scale
620
+ )
621
+ torch.cat([out_p, out_a], dim=2).float().square().mean().backward()
622
+
623
+ def sdpa_repeat_fwdbwd():
624
+ qq = q.detach().clone().requires_grad_(True)
625
+ kk = k.detach().clone().requires_grad_(True)
626
+ vv = v.detach().clone().requires_grad_(True)
627
+ kr, vr = _repeat_kv(kk), _repeat_kv(vv)
628
+ out_p = F.scaled_dot_product_attention(
629
+ qq[:, :, : args.prefix_len], kr, vr, attn_mask=add_mask[:, :, : args.prefix_len], scale=scale
630
+ )
631
+ kd = torch.cat([kr[:, :, : args.prefix_len].detach(), kr[:, :, args.prefix_len :]], dim=2)
632
+ vd = torch.cat([vr[:, :, : args.prefix_len].detach(), vr[:, :, args.prefix_len :]], dim=2)
633
+ out_a = F.scaled_dot_product_attention(
634
+ qq[:, :, args.prefix_len :], kd, vd, attn_mask=add_mask[:, :, args.prefix_len :], scale=scale
635
+ )
636
+ torch.cat([out_p, out_a], dim=2).float().square().mean().backward()
637
+
638
+ def package_fwdbwd():
639
+ qq = q.detach().clone().requires_grad_(True)
640
+ kk = k.detach().clone().requires_grad_(True)
641
+ vv = v.detach().clone().requires_grad_(True)
642
+ flex_ops.flex_attention(
643
+ qq,
644
+ kk,
645
+ vv,
646
+ prefix_len=args.prefix_len,
647
+ action_block_size=args.action_block_size,
648
+ prefix_valid=prefix_valid,
649
+ prefix_att=prefix_att,
650
+ scale=scale,
651
+ ).float().square().mean().backward()
652
+
653
+ def torch_flex_fwdbwd(preset: str, block_key: str):
654
+ if not flex_bundles:
655
+ raise RuntimeError("PyTorch FlexAttention is unavailable")
656
+ compiled_calls, block_prefix, block_action = flex_bundles[block_key]
657
+ prefix_call, action_call = compiled_calls[preset]
658
+ qq = q.detach().clone().requires_grad_(True)
659
+ kk = k.detach().clone().requires_grad_(True)
660
+ vv = v.detach().clone().requires_grad_(True)
661
+ out_p = prefix_call(
662
+ qq[:, :, : args.prefix_len],
663
+ kk,
664
+ vv,
665
+ block_prefix,
666
+ )
667
+ kd = torch.cat([kk[:, :, : args.prefix_len].detach(), kk[:, :, args.prefix_len :]], dim=2)
668
+ vd = torch.cat([vv[:, :, : args.prefix_len].detach(), vv[:, :, args.prefix_len :]], dim=2)
669
+ out_a = action_call(
670
+ qq[:, :, args.prefix_len :],
671
+ kd,
672
+ vd,
673
+ block_action,
674
+ )
675
+ torch.cat([out_p, out_a], dim=2).float().square().mean().backward()
676
+
677
+ report["sdpa_fwdbwd_ms"] = bench(sdpa_fwdbwd, args.warmup, args.iters)
678
+ if "package" in backends:
679
+ report["package_fwdbwd_ms"] = bench(package_fwdbwd, args.warmup, args.iters)
680
+ report["package_fwdbwd_speedup"] = report["sdpa_fwdbwd_ms"] / report["package_fwdbwd_ms"]
681
+ if "manual" in backends:
682
+ report["manual_fwdbwd_ms"] = bench(manual_fwdbwd, args.warmup, args.iters)
683
+ report["manual_fwdbwd_speedup"] = report["sdpa_fwdbwd_ms"] / report["manual_fwdbwd_ms"]
684
+ if gqa:
685
+ report["sdpa_repeat_fwdbwd_ms"] = bench(sdpa_repeat_fwdbwd, args.warmup, args.iters)
686
+ if "manual" in backends:
687
+ report["manual_eager_fwdbwd_ms"] = bench(manual_eager_fwdbwd, args.warmup, args.iters)
688
+ if flex_bundles:
689
+ report["torch_flex_fwdbwd_ms"] = {}
690
+ report["torch_flex_fwdbwd_speedup"] = {}
691
+ for block_key in flex_bundles:
692
+ report["torch_flex_fwdbwd_ms"][block_key] = {}
693
+ report["torch_flex_fwdbwd_speedup"][block_key] = {}
694
+ for block_key in flex_bundles:
695
+ for preset in flex_presets:
696
+ try:
697
+ ms = bench(lambda preset=preset, block_key=block_key: torch_flex_fwdbwd(preset, block_key), args.warmup, args.iters)
698
+ except Exception as exc:
699
+ report["torch_flex_fwdbwd_ms"][block_key][preset] = f"ERROR: {type(exc).__name__}: {exc}"
700
+ else:
701
+ report["torch_flex_fwdbwd_ms"][block_key][preset] = ms
702
+ report["torch_flex_fwdbwd_speedup"][block_key][preset] = report["sdpa_fwdbwd_ms"] / ms
703
+ report["best_torch_flex_fwdbwd"] = best_nested_speedup(report["torch_flex_fwdbwd_speedup"])
704
+
705
+ if "package" not in backends:
706
+ _finish(report, args)
707
+ return
708
+ q1 = q.detach().clone().requires_grad_(True)
709
+ k1 = k.detach().clone().requires_grad_(True)
710
+ v1 = v.detach().clone().requires_grad_(True)
711
+ q2 = q.detach().clone().requires_grad_(True)
712
+ k2 = k.detach().clone().requires_grad_(True)
713
+ v2 = v.detach().clone().requires_grad_(True)
714
+ out1_p = F.scaled_dot_product_attention(
715
+ q1[:, :, : args.prefix_len], k1, v1, attn_mask=add_mask[:, :, : args.prefix_len], scale=scale
716
+ )
717
+ k1d = torch.cat([k1[:, :, : args.prefix_len].detach(), k1[:, :, args.prefix_len :]], dim=2)
718
+ v1d = torch.cat([v1[:, :, : args.prefix_len].detach(), v1[:, :, args.prefix_len :]], dim=2)
719
+ out1_a = F.scaled_dot_product_attention(
720
+ q1[:, :, args.prefix_len :], k1d, v1d, attn_mask=add_mask[:, :, args.prefix_len :], scale=scale
721
+ )
722
+ torch.cat([out1_p, out1_a], dim=2).float().square().mean().backward()
723
+ flex_ops.flex_attention(
724
+ q2,
725
+ k2,
726
+ v2,
727
+ prefix_len=args.prefix_len,
728
+ action_block_size=args.action_block_size,
729
+ prefix_valid=prefix_valid,
730
+ prefix_att=prefix_att,
731
+ scale=scale,
732
+ ).float().square().mean().backward()
733
+ denom = torch.linalg.vector_norm(torch.cat([q1.grad.flatten(), k1.grad.flatten(), v1.grad.flatten()])).clamp_min(1e-12)
734
+ numer = torch.linalg.vector_norm(
735
+ torch.cat([(q1.grad - q2.grad).flatten(), (k1.grad - k2.grad).flatten(), (v1.grad - v2.grad).flatten()])
736
+ )
737
+ report["package_grad_norm_rel_diff"] = float(numer / denom)
738
+
739
+ _finish(report, args)
740
+
741
+
742
+ def _finish(report, args):
743
+ gates = {
744
+ "package_fwd_max_abs_diff": report.get("package_fwd_max_abs_diff", 0.0) <= 2e-3,
745
+ "package_grad_norm_rel_diff": report.get("package_grad_norm_rel_diff", 0.0) <= 1e-2,
746
+ }
747
+ if "torch_flex_fwd_max_abs_diff" in report:
748
+ vals = [v for by_block in report["torch_flex_fwd_max_abs_diff"].values() for v in by_block.values()]
749
+ gates["torch_flex_fwd_max_abs_diff"] = any(isinstance(v, float) and v <= 2e-3 for v in vals)
750
+ if "manual_fwd_max_abs_diff" in report:
751
+ gates["manual_fwd_max_abs_diff"] = report["manual_fwd_max_abs_diff"] <= 2e-3
752
+ if "manual_fwd_speedup" in report:
753
+ gates["manual_fwd_speedup_ge_1p0526"] = report["manual_fwd_speedup"] >= (1.0 / 0.95)
754
+ if "manual_fwdbwd_speedup" in report:
755
+ gates["manual_fwdbwd_speedup_ge_1p0526"] = report["manual_fwdbwd_speedup"] >= (1.0 / 0.95)
756
+ if "package_fwd_speedup" in report:
757
+ gates["package_fwd_speedup_ge_1p0526"] = report["package_fwd_speedup"] >= (1.0 / 0.95)
758
+ if report.get("sdpa_peak_bytes", 0) > 0:
759
+ gates["package_peak_memory_le_sdpa_plus_3pct"] = report["package_peak_bytes"] <= int(report["sdpa_peak_bytes"] * 1.03)
760
+ if "package_fwdbwd_speedup" in report:
761
+ gates["package_fwdbwd_speedup_ge_1p0526"] = report["package_fwdbwd_speedup"] >= (1.0 / 0.95)
762
+ if "torch_flex_fwd_speedup" in report:
763
+ vals = [v for by_block in report["torch_flex_fwd_speedup"].values() for v in by_block.values() if isinstance(v, float)]
764
+ gates["torch_flex_fwd_speedup_ge_1p0526"] = bool(vals) and max(vals) >= (1.0 / 0.95)
765
+ if "torch_flex_fwdbwd_speedup" in report:
766
+ vals = [v for by_block in report["torch_flex_fwdbwd_speedup"].values() for v in by_block.values() if isinstance(v, float)]
767
+ gates["torch_flex_fwdbwd_speedup_ge_1p0526"] = bool(vals) and max(vals) >= (1.0 / 0.95)
768
+ report["gates"] = gates
769
+
770
+ text = json.dumps(report, indent=2)
771
+ if args.output:
772
+ Path(args.output).write_text(text + "\n", encoding="utf-8")
773
+ print(text)
774
+ if args.require_gates and not all(gates.values()):
775
+ raise SystemExit("one or more Flex attention acceptance gates failed")
776
+
777
+
778
+ if __name__ == "__main__":
779
+ main()
build/torch211-cxx11-cu128-x86_64-linux/__init__.py ADDED
@@ -0,0 +1,487 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FlashRT Flex-style block-sparse attention training API.
2
+
3
+ The public function implements the PI052 prefix/action mask pattern:
4
+
5
+ * prefix query rows use the original K/V tensors, so prefix losses keep normal
6
+ gradients into prefix K/V;
7
+ * action query rows read detached prefix K/V plus normal action K/V by default,
8
+ matching the current training semantics.
9
+
10
+ Unsupported shapes route to the SDPA reference path. Native CUDA kernels are
11
+ not exposed until a shape-specialized implementation beats SDPA on the target
12
+ A100/5090 validation matrix.
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ from typing import Optional
18
+
19
+ import torch
20
+ import torch.nn.functional as F
21
+
22
+ try:
23
+ from ._ops import ops
24
+
25
+ _HAS_OPS = hasattr(ops, "_flashrt_training_package_marker")
26
+ except Exception: # source-tree tests before kernel-builder creates _ops.py
27
+ ops = None
28
+ _HAS_OPS = False
29
+
30
+
31
+ MASK_VALUE_F32 = -2.3819763e38
32
+
33
+
34
+ def _use_ops(namespace_ops) -> None:
35
+ """Install a manually built extension (dev/testing path)."""
36
+ global ops, _HAS_OPS
37
+ ops = namespace_ops
38
+ _HAS_OPS = hasattr(ops, "_flashrt_training_package_marker")
39
+
40
+
41
+ def _check_qkv(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor) -> None:
42
+ if q.dim() != 4 or k.dim() != 4 or v.dim() != 4:
43
+ raise ValueError("q, k, and v must be shaped (B, H, S, D)")
44
+ if q.shape[0] != k.shape[0] or q.shape[0] != v.shape[0]:
45
+ raise ValueError("q, k, and v batch dimensions must match")
46
+ if k.shape != v.shape:
47
+ raise ValueError("k and v shapes must match")
48
+ if q.shape[2] != k.shape[2] or q.shape[3] != k.shape[3]:
49
+ raise ValueError("q, k, and v sequence/head_dim dimensions must match")
50
+ if q.device != k.device or q.device != v.device:
51
+ raise ValueError("q, k, and v must be on the same device")
52
+
53
+
54
+ def _as_valid(mask: Optional[torch.Tensor], batch: int, length: int, device: torch.device) -> torch.Tensor:
55
+ if mask is None:
56
+ return torch.ones((batch, length), dtype=torch.bool, device=device)
57
+ if mask.shape != (batch, length):
58
+ raise ValueError(f"mask must be shaped {(batch, length)}, got {tuple(mask.shape)}")
59
+ return mask.to(device=device, dtype=torch.bool)
60
+
61
+
62
+ def build_block_sparse_bool_masks(
63
+ prefix_valid: Optional[torch.Tensor],
64
+ prefix_att: Optional[torch.Tensor],
65
+ *,
66
+ batch: int,
67
+ prefix_len: int,
68
+ action_len: int,
69
+ action_block_size: int,
70
+ non_fast_prefix_len: Optional[int] = None,
71
+ action_valid: Optional[torch.Tensor] = None,
72
+ device: Optional[torch.device] = None,
73
+ ) -> tuple[torch.Tensor, torch.Tensor]:
74
+ """Build boolean masks for the split FlexAttention SDPA calls.
75
+
76
+ Returns ``(prefix_rows, action_rows)`` with shapes ``(B, P, S)`` and
77
+ ``(B, A, S)``. Boolean True means the key/value position is visible.
78
+
79
+ ``prefix_att`` follows Lerobot's cumulative-block convention: prefix key
80
+ ``j`` is visible to prefix query ``i`` when ``cumsum(prefix_att)[j] <=
81
+ cumsum(prefix_att)[i]`` and both rows are valid. When omitted, prefix rows
82
+ attend to all valid prefix tokens.
83
+ """
84
+ if action_block_size <= 0:
85
+ raise ValueError("action_block_size must be positive")
86
+ if prefix_len < 0 or action_len < 0:
87
+ raise ValueError("prefix_len and action_len must be non-negative")
88
+ total_len = prefix_len + action_len
89
+ dev = device
90
+ if dev is None:
91
+ for t in (prefix_valid, prefix_att, action_valid):
92
+ if t is not None:
93
+ dev = t.device
94
+ break
95
+ if dev is None:
96
+ dev = torch.device("cpu")
97
+
98
+ p_valid = _as_valid(prefix_valid, batch, prefix_len, dev)
99
+ a_valid = _as_valid(action_valid, batch, action_len, dev)
100
+
101
+ if prefix_att is None:
102
+ prefix_rows = p_valid[:, :, None] & p_valid[:, None, :]
103
+ else:
104
+ if prefix_att.shape != (batch, prefix_len):
105
+ raise ValueError(
106
+ f"prefix_att must be shaped {(batch, prefix_len)}, got {tuple(prefix_att.shape)}"
107
+ )
108
+ cum = torch.cumsum(prefix_att.to(device=dev, dtype=torch.long), dim=1)
109
+ prefix_rows = (cum[:, None, :] <= cum[:, :, None]) & p_valid[:, :, None] & p_valid[:, None, :]
110
+
111
+ prefix_pad = torch.zeros((batch, prefix_len, action_len), dtype=torch.bool, device=dev)
112
+ prefix_rows = torch.cat([prefix_rows, prefix_pad], dim=2)
113
+
114
+ nf = prefix_len if non_fast_prefix_len is None else int(non_fast_prefix_len)
115
+ nf = max(0, min(nf, prefix_len))
116
+ action_to_prefix = torch.zeros((batch, action_len, prefix_len), dtype=torch.bool, device=dev)
117
+ if nf > 0:
118
+ action_to_prefix[:, :, :nf] = p_valid[:, None, :nf]
119
+ action_to_prefix &= a_valid[:, :, None]
120
+
121
+ q_block = torch.arange(action_len, device=dev) // int(action_block_size)
122
+ kv_block = q_block
123
+ action_block = q_block[:, None] == kv_block[None, :]
124
+ action_block = action_block[None, :, :].expand(batch, -1, -1)
125
+ action_block = action_block & a_valid[:, :, None] & a_valid[:, None, :]
126
+ action_rows = torch.cat([action_to_prefix, action_block], dim=2)
127
+
128
+ if prefix_rows.shape != (batch, prefix_len, total_len):
129
+ raise AssertionError("internal prefix mask shape error")
130
+ if action_rows.shape != (batch, action_len, total_len):
131
+ raise AssertionError("internal action mask shape error")
132
+ return prefix_rows, action_rows
133
+
134
+
135
+ def _bool_to_sdpa_mask(mask: torch.Tensor, q: torch.Tensor) -> torch.Tensor:
136
+ value = MASK_VALUE_F32
137
+ if q.dtype.is_floating_point:
138
+ finfo = torch.finfo(q.dtype)
139
+ value = max(MASK_VALUE_F32, finfo.min)
140
+ return torch.where(
141
+ mask[:, None, :, :],
142
+ torch.zeros((), dtype=q.dtype, device=q.device),
143
+ torch.full((), value, dtype=q.dtype, device=q.device),
144
+ )
145
+
146
+
147
+ def _slice_attention_mask(
148
+ attention_mask: torch.Tensor,
149
+ start: int,
150
+ end: int,
151
+ q: torch.Tensor,
152
+ ) -> torch.Tensor:
153
+ if attention_mask.dim() == 3:
154
+ mask = attention_mask[:, start:end, :]
155
+ if mask.dtype == torch.bool:
156
+ return mask[:, None, :, :]
157
+ return mask[:, None, :, :].to(dtype=q.dtype)
158
+ if attention_mask.dim() == 4:
159
+ mask = attention_mask[:, :, start:end, :]
160
+ return mask if mask.dtype == torch.bool else mask.to(dtype=q.dtype)
161
+ raise ValueError("attention_mask must be (B, S, S) or (B, 1|H, S, S)")
162
+
163
+
164
+ def _sdpa(
165
+ q: torch.Tensor,
166
+ k: torch.Tensor,
167
+ v: torch.Tensor,
168
+ mask: Optional[torch.Tensor],
169
+ *,
170
+ scale: Optional[float],
171
+ dropout_p: float,
172
+ enable_gqa: bool,
173
+ ) -> torch.Tensor:
174
+ kwargs = {"attn_mask": mask, "dropout_p": float(dropout_p), "scale": scale}
175
+ if enable_gqa:
176
+ kwargs["enable_gqa"] = True
177
+ try:
178
+ return F.scaled_dot_product_attention(q, k, v, **kwargs)
179
+ except TypeError:
180
+ kwargs.pop("enable_gqa", None)
181
+ return F.scaled_dot_product_attention(q, k, v, **kwargs)
182
+
183
+
184
+ def reference_flex_attention(
185
+ q: torch.Tensor,
186
+ k: torch.Tensor,
187
+ v: torch.Tensor,
188
+ *,
189
+ prefix_len: int,
190
+ action_block_size: int,
191
+ attention_mask: Optional[torch.Tensor] = None,
192
+ prefix_valid: Optional[torch.Tensor] = None,
193
+ prefix_att: Optional[torch.Tensor] = None,
194
+ non_fast_prefix_len: Optional[int] = None,
195
+ action_valid: Optional[torch.Tensor] = None,
196
+ detach_prefix_kv_for_action: bool = True,
197
+ scale: Optional[float] = None,
198
+ dropout_p: float = 0.0,
199
+ enable_gqa: Optional[bool] = None,
200
+ ) -> torch.Tensor:
201
+ """SDPA reference for the PI052 FlexAttention replacement shape.
202
+
203
+ Args:
204
+ q, k, v: ``(B, Hq/Hkv, S, D)`` tensors.
205
+ prefix_len: number of prefix rows/columns at the start of sequence.
206
+ action_block_size: size of each block-diagonal action segment.
207
+ attention_mask: optional prebuilt additive or boolean full mask.
208
+ prefix_valid: optional ``(B, P)`` valid prefix positions.
209
+ prefix_att: optional ``(B, P)`` cumulative-block markers.
210
+ non_fast_prefix_len: prefix columns visible to action rows.
211
+ action_valid: optional ``(B, A)`` valid action positions.
212
+ detach_prefix_kv_for_action: detach prefix K/V on the action-row path.
213
+ scale: SDPA scale. Defaults to ``D ** -0.5``.
214
+ dropout_p: SDPA dropout probability.
215
+ enable_gqa: pass SDPA GQA mode when q heads and kv heads differ.
216
+ """
217
+ _check_qkv(q, k, v)
218
+ batch, _, total_len, head_dim = q.shape
219
+ if not (0 <= int(prefix_len) <= total_len):
220
+ raise ValueError("prefix_len must be in [0, S]")
221
+ prefix_len = int(prefix_len)
222
+ action_len = total_len - prefix_len
223
+ if scale is None:
224
+ scale = head_dim**-0.5
225
+ if enable_gqa is None:
226
+ enable_gqa = q.shape[1] != k.shape[1]
227
+
228
+ q_prefix = q[:, :, :prefix_len, :]
229
+ q_action = q[:, :, prefix_len:, :]
230
+ k_prefix = k[:, :, :prefix_len, :]
231
+ k_action = k[:, :, prefix_len:, :]
232
+ v_prefix = v[:, :, :prefix_len, :]
233
+ v_action = v[:, :, prefix_len:, :]
234
+
235
+ if attention_mask is None:
236
+ prefix_bool, action_bool = build_block_sparse_bool_masks(
237
+ prefix_valid,
238
+ prefix_att,
239
+ batch=batch,
240
+ prefix_len=prefix_len,
241
+ action_len=action_len,
242
+ action_block_size=action_block_size,
243
+ non_fast_prefix_len=non_fast_prefix_len,
244
+ action_valid=action_valid,
245
+ device=q.device,
246
+ )
247
+ prefix_mask = _bool_to_sdpa_mask(prefix_bool, q)
248
+ action_mask = _bool_to_sdpa_mask(action_bool, q)
249
+ else:
250
+ prefix_mask = _slice_attention_mask(attention_mask, 0, prefix_len, q)
251
+ action_mask = _slice_attention_mask(attention_mask, prefix_len, total_len, q)
252
+
253
+ out_parts = []
254
+ if prefix_len:
255
+ out_parts.append(
256
+ _sdpa(
257
+ q_prefix,
258
+ k,
259
+ v,
260
+ prefix_mask,
261
+ scale=scale,
262
+ dropout_p=dropout_p,
263
+ enable_gqa=bool(enable_gqa),
264
+ )
265
+ )
266
+ if action_len:
267
+ prefix_k = k_prefix.detach() if detach_prefix_kv_for_action else k_prefix
268
+ prefix_v = v_prefix.detach() if detach_prefix_kv_for_action else v_prefix
269
+ k_for_action = torch.cat([prefix_k, k_action], dim=2)
270
+ v_for_action = torch.cat([prefix_v, v_action], dim=2)
271
+ out_parts.append(
272
+ _sdpa(
273
+ q_action,
274
+ k_for_action,
275
+ v_for_action,
276
+ action_mask,
277
+ scale=scale,
278
+ dropout_p=dropout_p,
279
+ enable_gqa=bool(enable_gqa),
280
+ )
281
+ )
282
+ if not out_parts:
283
+ return q.new_empty(q.shape)
284
+ return torch.cat(out_parts, dim=2) if len(out_parts) == 2 else out_parts[0]
285
+
286
+
287
+ def _manual_attention_part(qs, ks, vs, mask, scale):
288
+ """Materialized-logits attention part: cuBLAS GEMMs + fused masked softmax.
289
+
290
+ Same math as SDPA with an additive mask (fp32 softmax; logits stored in
291
+ the io dtype between the GEMM and the softmax). Grouped queries run as a
292
+ strided batched GEMM over the KV heads, so a 1-head K/V is never
293
+ repeated. At PI052 training shapes (GQA 8:1, D=256, bf16) this beats
294
+ both SDPA-with-dense-mask (2.3-3.1x) and the best FlexAttention
295
+ configuration (1.4-2.9x) on fwd+bwd — see benchmarks/RESULTS.md.
296
+ """
297
+ B, H, Sq, D = qs.shape
298
+ Hk = ks.shape[1]
299
+ if Hk != H:
300
+ g = H // Hk
301
+ q2 = qs.reshape(B, Hk, g * Sq, D)
302
+ logits = (q2 @ ks.transpose(-1, -2)).reshape(B, H, Sq, -1)
303
+ else:
304
+ logits = qs @ ks.transpose(-1, -2)
305
+ logits = logits * scale
306
+ if mask is not None:
307
+ logits = logits + mask
308
+ p = logits.float().softmax(dim=-1).to(qs.dtype)
309
+ if Hk != H:
310
+ out = (p.reshape(B, Hk, g * Sq, -1) @ vs).reshape(B, H, Sq, D)
311
+ else:
312
+ out = p @ vs
313
+ return out
314
+
315
+
316
+ _manual_part_compiled = None
317
+
318
+
319
+ def _get_manual_part():
320
+ global _manual_part_compiled
321
+ if _manual_part_compiled is None:
322
+ _manual_part_compiled = torch.compile(_manual_attention_part, dynamic=False)
323
+ return _manual_part_compiled
324
+
325
+
326
+ def manual_attention(
327
+ q: torch.Tensor,
328
+ k: torch.Tensor,
329
+ v: torch.Tensor,
330
+ *,
331
+ prefix_len: int,
332
+ action_block_size: int,
333
+ attention_mask: Optional[torch.Tensor] = None,
334
+ prefix_valid: Optional[torch.Tensor] = None,
335
+ prefix_att: Optional[torch.Tensor] = None,
336
+ non_fast_prefix_len: Optional[int] = None,
337
+ action_valid: Optional[torch.Tensor] = None,
338
+ detach_prefix_kv_for_action: bool = True,
339
+ scale: Optional[float] = None,
340
+ dropout_p: float = 0.0,
341
+ compile_part: bool = True,
342
+ ) -> torch.Tensor:
343
+ """Materialized-logits implementation of :func:`reference_flex_attention`.
344
+
345
+ Same mask semantics and prefix/action split; each part runs through
346
+ :func:`_manual_attention_part` instead of SDPA. ``dropout_p`` must be 0
347
+ (training attention dropout is unused in PI052); other values raise so
348
+ callers fall back explicitly.
349
+ """
350
+ if dropout_p:
351
+ raise ValueError("manual_attention does not support dropout; use the reference path")
352
+ _check_qkv(q, k, v)
353
+ batch, _, total_len, head_dim = q.shape
354
+ if not (0 <= int(prefix_len) <= total_len):
355
+ raise ValueError("prefix_len must be in [0, S]")
356
+ prefix_len = int(prefix_len)
357
+ action_len = total_len - prefix_len
358
+ if scale is None:
359
+ scale = head_dim**-0.5
360
+
361
+ if attention_mask is None:
362
+ prefix_bool, action_bool = build_block_sparse_bool_masks(
363
+ prefix_valid,
364
+ prefix_att,
365
+ batch=batch,
366
+ prefix_len=prefix_len,
367
+ action_len=action_len,
368
+ action_block_size=action_block_size,
369
+ non_fast_prefix_len=non_fast_prefix_len,
370
+ action_valid=action_valid,
371
+ device=q.device,
372
+ )
373
+ prefix_mask = _bool_to_sdpa_mask(prefix_bool, q)
374
+ action_mask = _bool_to_sdpa_mask(action_bool, q)
375
+ else:
376
+ prefix_mask = _slice_attention_mask(attention_mask, 0, prefix_len, q)
377
+ action_mask = _slice_attention_mask(attention_mask, prefix_len, total_len, q)
378
+ if prefix_mask.dtype == torch.bool:
379
+ prefix_mask = _bool_to_sdpa_mask(prefix_mask[:, 0], q)
380
+ if action_mask.dtype == torch.bool:
381
+ action_mask = _bool_to_sdpa_mask(action_mask[:, 0], q)
382
+
383
+ part = _get_manual_part() if compile_part else _manual_attention_part
384
+ out_parts = []
385
+ if prefix_len:
386
+ out_parts.append(part(q[:, :, :prefix_len, :], k, v, prefix_mask, scale))
387
+ if action_len:
388
+ k_prefix = k[:, :, :prefix_len, :]
389
+ v_prefix = v[:, :, :prefix_len, :]
390
+ if detach_prefix_kv_for_action:
391
+ k_prefix = k_prefix.detach()
392
+ v_prefix = v_prefix.detach()
393
+ k_for_action = torch.cat([k_prefix, k[:, :, prefix_len:, :]], dim=2)
394
+ v_for_action = torch.cat([v_prefix, v[:, :, prefix_len:, :]], dim=2)
395
+ out_parts.append(part(q[:, :, prefix_len:, :], k_for_action, v_for_action, action_mask, scale))
396
+ if not out_parts:
397
+ return q.new_empty(q.shape)
398
+ return torch.cat(out_parts, dim=2) if len(out_parts) == 2 else out_parts[0]
399
+
400
+
401
+ def flex_attention(
402
+ q: torch.Tensor,
403
+ k: torch.Tensor,
404
+ v: torch.Tensor,
405
+ *,
406
+ prefix_len: int,
407
+ action_block_size: int,
408
+ attention_mask: Optional[torch.Tensor] = None,
409
+ prefix_valid: Optional[torch.Tensor] = None,
410
+ prefix_att: Optional[torch.Tensor] = None,
411
+ non_fast_prefix_len: Optional[int] = None,
412
+ action_valid: Optional[torch.Tensor] = None,
413
+ detach_prefix_kv_for_action: bool = True,
414
+ scale: Optional[float] = None,
415
+ dropout_p: float = 0.0,
416
+ enable_gqa: Optional[bool] = None,
417
+ force_fallback: bool = False,
418
+ impl: str = "sdpa",
419
+ ) -> torch.Tensor:
420
+ """Flex-style block-sparse attention.
421
+
422
+ ``impl="sdpa"`` (default) keeps the SDPA reference path;
423
+ ``impl="manual"`` routes through the materialized-logits
424
+ implementation; ``impl="auto"`` picks manual only where it has been
425
+ measured to win end-to-end — consumer Blackwell (sm120-class) with
426
+ no dropout. On A100 (sm80) the manual math wins microbenches but
427
+ loses training-step integration, and on H100/H200 (sm90) the fused
428
+ FMHA kernels win outright, so auto keeps SDPA there.
429
+ """
430
+ _ = force_fallback
431
+ if impl == "auto":
432
+ sm120 = q.is_cuda and torch.cuda.get_device_capability(q.device)[0] == 12
433
+ impl = "manual" if (sm120 and not dropout_p) else "sdpa"
434
+ if impl == "manual":
435
+ return manual_attention(
436
+ q,
437
+ k,
438
+ v,
439
+ prefix_len=prefix_len,
440
+ action_block_size=action_block_size,
441
+ attention_mask=attention_mask,
442
+ prefix_valid=prefix_valid,
443
+ prefix_att=prefix_att,
444
+ non_fast_prefix_len=non_fast_prefix_len,
445
+ action_valid=action_valid,
446
+ detach_prefix_kv_for_action=detach_prefix_kv_for_action,
447
+ scale=scale,
448
+ dropout_p=dropout_p,
449
+ )
450
+ return reference_flex_attention(
451
+ q,
452
+ k,
453
+ v,
454
+ prefix_len=prefix_len,
455
+ action_block_size=action_block_size,
456
+ attention_mask=attention_mask,
457
+ prefix_valid=prefix_valid,
458
+ prefix_att=prefix_att,
459
+ non_fast_prefix_len=non_fast_prefix_len,
460
+ action_valid=action_valid,
461
+ detach_prefix_kv_for_action=detach_prefix_kv_for_action,
462
+ scale=scale,
463
+ dropout_p=dropout_p,
464
+ enable_gqa=enable_gqa,
465
+ )
466
+
467
+
468
+ def flex_attention_forward(*args, **kwargs) -> torch.Tensor:
469
+ """Forward-only compatibility wrapper."""
470
+ return flex_attention(*args, **kwargs)
471
+
472
+
473
+ def backend_marker(x: torch.Tensor) -> torch.Tensor:
474
+ if ops is None:
475
+ return x
476
+ return ops._flashrt_training_package_marker(x)
477
+
478
+
479
+ __all__ = [
480
+ "MASK_VALUE_F32",
481
+ "backend_marker",
482
+ "build_block_sparse_bool_masks",
483
+ "flex_attention",
484
+ "flex_attention_forward",
485
+ "manual_attention",
486
+ "reference_flex_attention",
487
+ ]
build/torch211-cxx11-cu128-x86_64-linux/_flashrt_flex_attention_train_cuda_eaf7454.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0c55e2302f1031c5b9071ba00e373d2ef6f63f3b5f2c2c3cab80cf91806a6ed5
3
+ size 52792
build/torch211-cxx11-cu128-x86_64-linux/_ops.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _flashrt_flex_attention_train_cuda_eaf7454
3
+ ops = torch.ops._flashrt_flex_attention_train_cuda_eaf7454
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ """
7
+ Prefix op by namespace.
8
+ """
9
+ return f"_flashrt_flex_attention_train_cuda_eaf7454::{op_name}"
build/torch211-cxx11-cu128-x86_64-linux/flashrt_flex_attention_train/__init__.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import ctypes
2
+ import importlib.util
3
+ import sys
4
+ from pathlib import Path
5
+ from types import ModuleType
6
+
7
+
8
+ def _import_from_path(file_path: Path) -> ModuleType:
9
+ # We cannot use the module name as-is, after adding it to `sys.modules`,
10
+ # it would also be used for other imports. So, we make a module name that
11
+ # depends on the path for it to be unique using the hex-encoded hash of
12
+ # the path.
13
+ path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
14
+ module_name = path_hash
15
+ spec = importlib.util.spec_from_file_location(module_name, file_path)
16
+ if spec is None:
17
+ raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
18
+ module = importlib.util.module_from_spec(spec)
19
+ if module is None:
20
+ raise ImportError(f"Cannot load module {module_name} from spec")
21
+ sys.modules[module_name] = module
22
+ spec.loader.exec_module(module) # type: ignore
23
+ return module
24
+
25
+
26
+ globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
build/torch211-cxx11-cu128-x86_64-linux/metadata.json ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "flashrt-flex-attention-train",
3
+ "id": "_flashrt_flex_attention_train_cuda_eaf7454",
4
+ "version": 1,
5
+ "license": "Apache-2.0",
6
+ "python-depends": [],
7
+ "backend": {
8
+ "type": "cuda",
9
+ "archs": [
10
+ "10.0",
11
+ "10.1",
12
+ "12.0+PTX",
13
+ "7.0",
14
+ "7.2",
15
+ "7.5",
16
+ "8.0",
17
+ "8.6",
18
+ "8.7",
19
+ "8.9",
20
+ "9.0"
21
+ ]
22
+ }
23
+ }
build/torch211-cxx11-cu130-x86_64-linux/__init__.py ADDED
@@ -0,0 +1,487 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FlashRT Flex-style block-sparse attention training API.
2
+
3
+ The public function implements the PI052 prefix/action mask pattern:
4
+
5
+ * prefix query rows use the original K/V tensors, so prefix losses keep normal
6
+ gradients into prefix K/V;
7
+ * action query rows read detached prefix K/V plus normal action K/V by default,
8
+ matching the current training semantics.
9
+
10
+ Unsupported shapes route to the SDPA reference path. Native CUDA kernels are
11
+ not exposed until a shape-specialized implementation beats SDPA on the target
12
+ A100/5090 validation matrix.
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ from typing import Optional
18
+
19
+ import torch
20
+ import torch.nn.functional as F
21
+
22
+ try:
23
+ from ._ops import ops
24
+
25
+ _HAS_OPS = hasattr(ops, "_flashrt_training_package_marker")
26
+ except Exception: # source-tree tests before kernel-builder creates _ops.py
27
+ ops = None
28
+ _HAS_OPS = False
29
+
30
+
31
+ MASK_VALUE_F32 = -2.3819763e38
32
+
33
+
34
+ def _use_ops(namespace_ops) -> None:
35
+ """Install a manually built extension (dev/testing path)."""
36
+ global ops, _HAS_OPS
37
+ ops = namespace_ops
38
+ _HAS_OPS = hasattr(ops, "_flashrt_training_package_marker")
39
+
40
+
41
+ def _check_qkv(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor) -> None:
42
+ if q.dim() != 4 or k.dim() != 4 or v.dim() != 4:
43
+ raise ValueError("q, k, and v must be shaped (B, H, S, D)")
44
+ if q.shape[0] != k.shape[0] or q.shape[0] != v.shape[0]:
45
+ raise ValueError("q, k, and v batch dimensions must match")
46
+ if k.shape != v.shape:
47
+ raise ValueError("k and v shapes must match")
48
+ if q.shape[2] != k.shape[2] or q.shape[3] != k.shape[3]:
49
+ raise ValueError("q, k, and v sequence/head_dim dimensions must match")
50
+ if q.device != k.device or q.device != v.device:
51
+ raise ValueError("q, k, and v must be on the same device")
52
+
53
+
54
+ def _as_valid(mask: Optional[torch.Tensor], batch: int, length: int, device: torch.device) -> torch.Tensor:
55
+ if mask is None:
56
+ return torch.ones((batch, length), dtype=torch.bool, device=device)
57
+ if mask.shape != (batch, length):
58
+ raise ValueError(f"mask must be shaped {(batch, length)}, got {tuple(mask.shape)}")
59
+ return mask.to(device=device, dtype=torch.bool)
60
+
61
+
62
+ def build_block_sparse_bool_masks(
63
+ prefix_valid: Optional[torch.Tensor],
64
+ prefix_att: Optional[torch.Tensor],
65
+ *,
66
+ batch: int,
67
+ prefix_len: int,
68
+ action_len: int,
69
+ action_block_size: int,
70
+ non_fast_prefix_len: Optional[int] = None,
71
+ action_valid: Optional[torch.Tensor] = None,
72
+ device: Optional[torch.device] = None,
73
+ ) -> tuple[torch.Tensor, torch.Tensor]:
74
+ """Build boolean masks for the split FlexAttention SDPA calls.
75
+
76
+ Returns ``(prefix_rows, action_rows)`` with shapes ``(B, P, S)`` and
77
+ ``(B, A, S)``. Boolean True means the key/value position is visible.
78
+
79
+ ``prefix_att`` follows Lerobot's cumulative-block convention: prefix key
80
+ ``j`` is visible to prefix query ``i`` when ``cumsum(prefix_att)[j] <=
81
+ cumsum(prefix_att)[i]`` and both rows are valid. When omitted, prefix rows
82
+ attend to all valid prefix tokens.
83
+ """
84
+ if action_block_size <= 0:
85
+ raise ValueError("action_block_size must be positive")
86
+ if prefix_len < 0 or action_len < 0:
87
+ raise ValueError("prefix_len and action_len must be non-negative")
88
+ total_len = prefix_len + action_len
89
+ dev = device
90
+ if dev is None:
91
+ for t in (prefix_valid, prefix_att, action_valid):
92
+ if t is not None:
93
+ dev = t.device
94
+ break
95
+ if dev is None:
96
+ dev = torch.device("cpu")
97
+
98
+ p_valid = _as_valid(prefix_valid, batch, prefix_len, dev)
99
+ a_valid = _as_valid(action_valid, batch, action_len, dev)
100
+
101
+ if prefix_att is None:
102
+ prefix_rows = p_valid[:, :, None] & p_valid[:, None, :]
103
+ else:
104
+ if prefix_att.shape != (batch, prefix_len):
105
+ raise ValueError(
106
+ f"prefix_att must be shaped {(batch, prefix_len)}, got {tuple(prefix_att.shape)}"
107
+ )
108
+ cum = torch.cumsum(prefix_att.to(device=dev, dtype=torch.long), dim=1)
109
+ prefix_rows = (cum[:, None, :] <= cum[:, :, None]) & p_valid[:, :, None] & p_valid[:, None, :]
110
+
111
+ prefix_pad = torch.zeros((batch, prefix_len, action_len), dtype=torch.bool, device=dev)
112
+ prefix_rows = torch.cat([prefix_rows, prefix_pad], dim=2)
113
+
114
+ nf = prefix_len if non_fast_prefix_len is None else int(non_fast_prefix_len)
115
+ nf = max(0, min(nf, prefix_len))
116
+ action_to_prefix = torch.zeros((batch, action_len, prefix_len), dtype=torch.bool, device=dev)
117
+ if nf > 0:
118
+ action_to_prefix[:, :, :nf] = p_valid[:, None, :nf]
119
+ action_to_prefix &= a_valid[:, :, None]
120
+
121
+ q_block = torch.arange(action_len, device=dev) // int(action_block_size)
122
+ kv_block = q_block
123
+ action_block = q_block[:, None] == kv_block[None, :]
124
+ action_block = action_block[None, :, :].expand(batch, -1, -1)
125
+ action_block = action_block & a_valid[:, :, None] & a_valid[:, None, :]
126
+ action_rows = torch.cat([action_to_prefix, action_block], dim=2)
127
+
128
+ if prefix_rows.shape != (batch, prefix_len, total_len):
129
+ raise AssertionError("internal prefix mask shape error")
130
+ if action_rows.shape != (batch, action_len, total_len):
131
+ raise AssertionError("internal action mask shape error")
132
+ return prefix_rows, action_rows
133
+
134
+
135
+ def _bool_to_sdpa_mask(mask: torch.Tensor, q: torch.Tensor) -> torch.Tensor:
136
+ value = MASK_VALUE_F32
137
+ if q.dtype.is_floating_point:
138
+ finfo = torch.finfo(q.dtype)
139
+ value = max(MASK_VALUE_F32, finfo.min)
140
+ return torch.where(
141
+ mask[:, None, :, :],
142
+ torch.zeros((), dtype=q.dtype, device=q.device),
143
+ torch.full((), value, dtype=q.dtype, device=q.device),
144
+ )
145
+
146
+
147
+ def _slice_attention_mask(
148
+ attention_mask: torch.Tensor,
149
+ start: int,
150
+ end: int,
151
+ q: torch.Tensor,
152
+ ) -> torch.Tensor:
153
+ if attention_mask.dim() == 3:
154
+ mask = attention_mask[:, start:end, :]
155
+ if mask.dtype == torch.bool:
156
+ return mask[:, None, :, :]
157
+ return mask[:, None, :, :].to(dtype=q.dtype)
158
+ if attention_mask.dim() == 4:
159
+ mask = attention_mask[:, :, start:end, :]
160
+ return mask if mask.dtype == torch.bool else mask.to(dtype=q.dtype)
161
+ raise ValueError("attention_mask must be (B, S, S) or (B, 1|H, S, S)")
162
+
163
+
164
+ def _sdpa(
165
+ q: torch.Tensor,
166
+ k: torch.Tensor,
167
+ v: torch.Tensor,
168
+ mask: Optional[torch.Tensor],
169
+ *,
170
+ scale: Optional[float],
171
+ dropout_p: float,
172
+ enable_gqa: bool,
173
+ ) -> torch.Tensor:
174
+ kwargs = {"attn_mask": mask, "dropout_p": float(dropout_p), "scale": scale}
175
+ if enable_gqa:
176
+ kwargs["enable_gqa"] = True
177
+ try:
178
+ return F.scaled_dot_product_attention(q, k, v, **kwargs)
179
+ except TypeError:
180
+ kwargs.pop("enable_gqa", None)
181
+ return F.scaled_dot_product_attention(q, k, v, **kwargs)
182
+
183
+
184
+ def reference_flex_attention(
185
+ q: torch.Tensor,
186
+ k: torch.Tensor,
187
+ v: torch.Tensor,
188
+ *,
189
+ prefix_len: int,
190
+ action_block_size: int,
191
+ attention_mask: Optional[torch.Tensor] = None,
192
+ prefix_valid: Optional[torch.Tensor] = None,
193
+ prefix_att: Optional[torch.Tensor] = None,
194
+ non_fast_prefix_len: Optional[int] = None,
195
+ action_valid: Optional[torch.Tensor] = None,
196
+ detach_prefix_kv_for_action: bool = True,
197
+ scale: Optional[float] = None,
198
+ dropout_p: float = 0.0,
199
+ enable_gqa: Optional[bool] = None,
200
+ ) -> torch.Tensor:
201
+ """SDPA reference for the PI052 FlexAttention replacement shape.
202
+
203
+ Args:
204
+ q, k, v: ``(B, Hq/Hkv, S, D)`` tensors.
205
+ prefix_len: number of prefix rows/columns at the start of sequence.
206
+ action_block_size: size of each block-diagonal action segment.
207
+ attention_mask: optional prebuilt additive or boolean full mask.
208
+ prefix_valid: optional ``(B, P)`` valid prefix positions.
209
+ prefix_att: optional ``(B, P)`` cumulative-block markers.
210
+ non_fast_prefix_len: prefix columns visible to action rows.
211
+ action_valid: optional ``(B, A)`` valid action positions.
212
+ detach_prefix_kv_for_action: detach prefix K/V on the action-row path.
213
+ scale: SDPA scale. Defaults to ``D ** -0.5``.
214
+ dropout_p: SDPA dropout probability.
215
+ enable_gqa: pass SDPA GQA mode when q heads and kv heads differ.
216
+ """
217
+ _check_qkv(q, k, v)
218
+ batch, _, total_len, head_dim = q.shape
219
+ if not (0 <= int(prefix_len) <= total_len):
220
+ raise ValueError("prefix_len must be in [0, S]")
221
+ prefix_len = int(prefix_len)
222
+ action_len = total_len - prefix_len
223
+ if scale is None:
224
+ scale = head_dim**-0.5
225
+ if enable_gqa is None:
226
+ enable_gqa = q.shape[1] != k.shape[1]
227
+
228
+ q_prefix = q[:, :, :prefix_len, :]
229
+ q_action = q[:, :, prefix_len:, :]
230
+ k_prefix = k[:, :, :prefix_len, :]
231
+ k_action = k[:, :, prefix_len:, :]
232
+ v_prefix = v[:, :, :prefix_len, :]
233
+ v_action = v[:, :, prefix_len:, :]
234
+
235
+ if attention_mask is None:
236
+ prefix_bool, action_bool = build_block_sparse_bool_masks(
237
+ prefix_valid,
238
+ prefix_att,
239
+ batch=batch,
240
+ prefix_len=prefix_len,
241
+ action_len=action_len,
242
+ action_block_size=action_block_size,
243
+ non_fast_prefix_len=non_fast_prefix_len,
244
+ action_valid=action_valid,
245
+ device=q.device,
246
+ )
247
+ prefix_mask = _bool_to_sdpa_mask(prefix_bool, q)
248
+ action_mask = _bool_to_sdpa_mask(action_bool, q)
249
+ else:
250
+ prefix_mask = _slice_attention_mask(attention_mask, 0, prefix_len, q)
251
+ action_mask = _slice_attention_mask(attention_mask, prefix_len, total_len, q)
252
+
253
+ out_parts = []
254
+ if prefix_len:
255
+ out_parts.append(
256
+ _sdpa(
257
+ q_prefix,
258
+ k,
259
+ v,
260
+ prefix_mask,
261
+ scale=scale,
262
+ dropout_p=dropout_p,
263
+ enable_gqa=bool(enable_gqa),
264
+ )
265
+ )
266
+ if action_len:
267
+ prefix_k = k_prefix.detach() if detach_prefix_kv_for_action else k_prefix
268
+ prefix_v = v_prefix.detach() if detach_prefix_kv_for_action else v_prefix
269
+ k_for_action = torch.cat([prefix_k, k_action], dim=2)
270
+ v_for_action = torch.cat([prefix_v, v_action], dim=2)
271
+ out_parts.append(
272
+ _sdpa(
273
+ q_action,
274
+ k_for_action,
275
+ v_for_action,
276
+ action_mask,
277
+ scale=scale,
278
+ dropout_p=dropout_p,
279
+ enable_gqa=bool(enable_gqa),
280
+ )
281
+ )
282
+ if not out_parts:
283
+ return q.new_empty(q.shape)
284
+ return torch.cat(out_parts, dim=2) if len(out_parts) == 2 else out_parts[0]
285
+
286
+
287
+ def _manual_attention_part(qs, ks, vs, mask, scale):
288
+ """Materialized-logits attention part: cuBLAS GEMMs + fused masked softmax.
289
+
290
+ Same math as SDPA with an additive mask (fp32 softmax; logits stored in
291
+ the io dtype between the GEMM and the softmax). Grouped queries run as a
292
+ strided batched GEMM over the KV heads, so a 1-head K/V is never
293
+ repeated. At PI052 training shapes (GQA 8:1, D=256, bf16) this beats
294
+ both SDPA-with-dense-mask (2.3-3.1x) and the best FlexAttention
295
+ configuration (1.4-2.9x) on fwd+bwd — see benchmarks/RESULTS.md.
296
+ """
297
+ B, H, Sq, D = qs.shape
298
+ Hk = ks.shape[1]
299
+ if Hk != H:
300
+ g = H // Hk
301
+ q2 = qs.reshape(B, Hk, g * Sq, D)
302
+ logits = (q2 @ ks.transpose(-1, -2)).reshape(B, H, Sq, -1)
303
+ else:
304
+ logits = qs @ ks.transpose(-1, -2)
305
+ logits = logits * scale
306
+ if mask is not None:
307
+ logits = logits + mask
308
+ p = logits.float().softmax(dim=-1).to(qs.dtype)
309
+ if Hk != H:
310
+ out = (p.reshape(B, Hk, g * Sq, -1) @ vs).reshape(B, H, Sq, D)
311
+ else:
312
+ out = p @ vs
313
+ return out
314
+
315
+
316
+ _manual_part_compiled = None
317
+
318
+
319
+ def _get_manual_part():
320
+ global _manual_part_compiled
321
+ if _manual_part_compiled is None:
322
+ _manual_part_compiled = torch.compile(_manual_attention_part, dynamic=False)
323
+ return _manual_part_compiled
324
+
325
+
326
+ def manual_attention(
327
+ q: torch.Tensor,
328
+ k: torch.Tensor,
329
+ v: torch.Tensor,
330
+ *,
331
+ prefix_len: int,
332
+ action_block_size: int,
333
+ attention_mask: Optional[torch.Tensor] = None,
334
+ prefix_valid: Optional[torch.Tensor] = None,
335
+ prefix_att: Optional[torch.Tensor] = None,
336
+ non_fast_prefix_len: Optional[int] = None,
337
+ action_valid: Optional[torch.Tensor] = None,
338
+ detach_prefix_kv_for_action: bool = True,
339
+ scale: Optional[float] = None,
340
+ dropout_p: float = 0.0,
341
+ compile_part: bool = True,
342
+ ) -> torch.Tensor:
343
+ """Materialized-logits implementation of :func:`reference_flex_attention`.
344
+
345
+ Same mask semantics and prefix/action split; each part runs through
346
+ :func:`_manual_attention_part` instead of SDPA. ``dropout_p`` must be 0
347
+ (training attention dropout is unused in PI052); other values raise so
348
+ callers fall back explicitly.
349
+ """
350
+ if dropout_p:
351
+ raise ValueError("manual_attention does not support dropout; use the reference path")
352
+ _check_qkv(q, k, v)
353
+ batch, _, total_len, head_dim = q.shape
354
+ if not (0 <= int(prefix_len) <= total_len):
355
+ raise ValueError("prefix_len must be in [0, S]")
356
+ prefix_len = int(prefix_len)
357
+ action_len = total_len - prefix_len
358
+ if scale is None:
359
+ scale = head_dim**-0.5
360
+
361
+ if attention_mask is None:
362
+ prefix_bool, action_bool = build_block_sparse_bool_masks(
363
+ prefix_valid,
364
+ prefix_att,
365
+ batch=batch,
366
+ prefix_len=prefix_len,
367
+ action_len=action_len,
368
+ action_block_size=action_block_size,
369
+ non_fast_prefix_len=non_fast_prefix_len,
370
+ action_valid=action_valid,
371
+ device=q.device,
372
+ )
373
+ prefix_mask = _bool_to_sdpa_mask(prefix_bool, q)
374
+ action_mask = _bool_to_sdpa_mask(action_bool, q)
375
+ else:
376
+ prefix_mask = _slice_attention_mask(attention_mask, 0, prefix_len, q)
377
+ action_mask = _slice_attention_mask(attention_mask, prefix_len, total_len, q)
378
+ if prefix_mask.dtype == torch.bool:
379
+ prefix_mask = _bool_to_sdpa_mask(prefix_mask[:, 0], q)
380
+ if action_mask.dtype == torch.bool:
381
+ action_mask = _bool_to_sdpa_mask(action_mask[:, 0], q)
382
+
383
+ part = _get_manual_part() if compile_part else _manual_attention_part
384
+ out_parts = []
385
+ if prefix_len:
386
+ out_parts.append(part(q[:, :, :prefix_len, :], k, v, prefix_mask, scale))
387
+ if action_len:
388
+ k_prefix = k[:, :, :prefix_len, :]
389
+ v_prefix = v[:, :, :prefix_len, :]
390
+ if detach_prefix_kv_for_action:
391
+ k_prefix = k_prefix.detach()
392
+ v_prefix = v_prefix.detach()
393
+ k_for_action = torch.cat([k_prefix, k[:, :, prefix_len:, :]], dim=2)
394
+ v_for_action = torch.cat([v_prefix, v[:, :, prefix_len:, :]], dim=2)
395
+ out_parts.append(part(q[:, :, prefix_len:, :], k_for_action, v_for_action, action_mask, scale))
396
+ if not out_parts:
397
+ return q.new_empty(q.shape)
398
+ return torch.cat(out_parts, dim=2) if len(out_parts) == 2 else out_parts[0]
399
+
400
+
401
+ def flex_attention(
402
+ q: torch.Tensor,
403
+ k: torch.Tensor,
404
+ v: torch.Tensor,
405
+ *,
406
+ prefix_len: int,
407
+ action_block_size: int,
408
+ attention_mask: Optional[torch.Tensor] = None,
409
+ prefix_valid: Optional[torch.Tensor] = None,
410
+ prefix_att: Optional[torch.Tensor] = None,
411
+ non_fast_prefix_len: Optional[int] = None,
412
+ action_valid: Optional[torch.Tensor] = None,
413
+ detach_prefix_kv_for_action: bool = True,
414
+ scale: Optional[float] = None,
415
+ dropout_p: float = 0.0,
416
+ enable_gqa: Optional[bool] = None,
417
+ force_fallback: bool = False,
418
+ impl: str = "sdpa",
419
+ ) -> torch.Tensor:
420
+ """Flex-style block-sparse attention.
421
+
422
+ ``impl="sdpa"`` (default) keeps the SDPA reference path;
423
+ ``impl="manual"`` routes through the materialized-logits
424
+ implementation; ``impl="auto"`` picks manual only where it has been
425
+ measured to win end-to-end — consumer Blackwell (sm120-class) with
426
+ no dropout. On A100 (sm80) the manual math wins microbenches but
427
+ loses training-step integration, and on H100/H200 (sm90) the fused
428
+ FMHA kernels win outright, so auto keeps SDPA there.
429
+ """
430
+ _ = force_fallback
431
+ if impl == "auto":
432
+ sm120 = q.is_cuda and torch.cuda.get_device_capability(q.device)[0] == 12
433
+ impl = "manual" if (sm120 and not dropout_p) else "sdpa"
434
+ if impl == "manual":
435
+ return manual_attention(
436
+ q,
437
+ k,
438
+ v,
439
+ prefix_len=prefix_len,
440
+ action_block_size=action_block_size,
441
+ attention_mask=attention_mask,
442
+ prefix_valid=prefix_valid,
443
+ prefix_att=prefix_att,
444
+ non_fast_prefix_len=non_fast_prefix_len,
445
+ action_valid=action_valid,
446
+ detach_prefix_kv_for_action=detach_prefix_kv_for_action,
447
+ scale=scale,
448
+ dropout_p=dropout_p,
449
+ )
450
+ return reference_flex_attention(
451
+ q,
452
+ k,
453
+ v,
454
+ prefix_len=prefix_len,
455
+ action_block_size=action_block_size,
456
+ attention_mask=attention_mask,
457
+ prefix_valid=prefix_valid,
458
+ prefix_att=prefix_att,
459
+ non_fast_prefix_len=non_fast_prefix_len,
460
+ action_valid=action_valid,
461
+ detach_prefix_kv_for_action=detach_prefix_kv_for_action,
462
+ scale=scale,
463
+ dropout_p=dropout_p,
464
+ enable_gqa=enable_gqa,
465
+ )
466
+
467
+
468
+ def flex_attention_forward(*args, **kwargs) -> torch.Tensor:
469
+ """Forward-only compatibility wrapper."""
470
+ return flex_attention(*args, **kwargs)
471
+
472
+
473
+ def backend_marker(x: torch.Tensor) -> torch.Tensor:
474
+ if ops is None:
475
+ return x
476
+ return ops._flashrt_training_package_marker(x)
477
+
478
+
479
+ __all__ = [
480
+ "MASK_VALUE_F32",
481
+ "backend_marker",
482
+ "build_block_sparse_bool_masks",
483
+ "flex_attention",
484
+ "flex_attention_forward",
485
+ "manual_attention",
486
+ "reference_flex_attention",
487
+ ]
build/torch211-cxx11-cu130-x86_64-linux/_flashrt_flex_attention_train_cuda_eaf7454.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9aa4c1d74ad4357d787307b8980e4cd4e0641b45d1b41e3d8285955f1eeb4d0c
3
+ size 56888
build/torch211-cxx11-cu130-x86_64-linux/_ops.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _flashrt_flex_attention_train_cuda_eaf7454
3
+ ops = torch.ops._flashrt_flex_attention_train_cuda_eaf7454
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ """
7
+ Prefix op by namespace.
8
+ """
9
+ return f"_flashrt_flex_attention_train_cuda_eaf7454::{op_name}"
build/torch211-cxx11-cu130-x86_64-linux/flashrt_flex_attention_train/__init__.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import ctypes
2
+ import importlib.util
3
+ import sys
4
+ from pathlib import Path
5
+ from types import ModuleType
6
+
7
+
8
+ def _import_from_path(file_path: Path) -> ModuleType:
9
+ # We cannot use the module name as-is, after adding it to `sys.modules`,
10
+ # it would also be used for other imports. So, we make a module name that
11
+ # depends on the path for it to be unique using the hex-encoded hash of
12
+ # the path.
13
+ path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
14
+ module_name = path_hash
15
+ spec = importlib.util.spec_from_file_location(module_name, file_path)
16
+ if spec is None:
17
+ raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
18
+ module = importlib.util.module_from_spec(spec)
19
+ if module is None:
20
+ raise ImportError(f"Cannot load module {module_name} from spec")
21
+ sys.modules[module_name] = module
22
+ spec.loader.exec_module(module) # type: ignore
23
+ return module
24
+
25
+
26
+ globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
build/torch211-cxx11-cu130-x86_64-linux/metadata.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "flashrt-flex-attention-train",
3
+ "id": "_flashrt_flex_attention_train_cuda_eaf7454",
4
+ "version": 1,
5
+ "license": "Apache-2.0",
6
+ "python-depends": [],
7
+ "backend": {
8
+ "type": "cuda",
9
+ "archs": [
10
+ "10.0",
11
+ "11.0",
12
+ "12.0",
13
+ "12.1+PTX",
14
+ "7.5",
15
+ "8.0",
16
+ "8.6",
17
+ "8.7",
18
+ "8.9",
19
+ "9.0"
20
+ ]
21
+ }
22
+ }
build/torch212-cxx11-cu130-x86_64-linux/__init__.py ADDED
@@ -0,0 +1,487 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FlashRT Flex-style block-sparse attention training API.
2
+
3
+ The public function implements the PI052 prefix/action mask pattern:
4
+
5
+ * prefix query rows use the original K/V tensors, so prefix losses keep normal
6
+ gradients into prefix K/V;
7
+ * action query rows read detached prefix K/V plus normal action K/V by default,
8
+ matching the current training semantics.
9
+
10
+ Unsupported shapes route to the SDPA reference path. Native CUDA kernels are
11
+ not exposed until a shape-specialized implementation beats SDPA on the target
12
+ A100/5090 validation matrix.
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ from typing import Optional
18
+
19
+ import torch
20
+ import torch.nn.functional as F
21
+
22
+ try:
23
+ from ._ops import ops
24
+
25
+ _HAS_OPS = hasattr(ops, "_flashrt_training_package_marker")
26
+ except Exception: # source-tree tests before kernel-builder creates _ops.py
27
+ ops = None
28
+ _HAS_OPS = False
29
+
30
+
31
+ MASK_VALUE_F32 = -2.3819763e38
32
+
33
+
34
+ def _use_ops(namespace_ops) -> None:
35
+ """Install a manually built extension (dev/testing path)."""
36
+ global ops, _HAS_OPS
37
+ ops = namespace_ops
38
+ _HAS_OPS = hasattr(ops, "_flashrt_training_package_marker")
39
+
40
+
41
+ def _check_qkv(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor) -> None:
42
+ if q.dim() != 4 or k.dim() != 4 or v.dim() != 4:
43
+ raise ValueError("q, k, and v must be shaped (B, H, S, D)")
44
+ if q.shape[0] != k.shape[0] or q.shape[0] != v.shape[0]:
45
+ raise ValueError("q, k, and v batch dimensions must match")
46
+ if k.shape != v.shape:
47
+ raise ValueError("k and v shapes must match")
48
+ if q.shape[2] != k.shape[2] or q.shape[3] != k.shape[3]:
49
+ raise ValueError("q, k, and v sequence/head_dim dimensions must match")
50
+ if q.device != k.device or q.device != v.device:
51
+ raise ValueError("q, k, and v must be on the same device")
52
+
53
+
54
+ def _as_valid(mask: Optional[torch.Tensor], batch: int, length: int, device: torch.device) -> torch.Tensor:
55
+ if mask is None:
56
+ return torch.ones((batch, length), dtype=torch.bool, device=device)
57
+ if mask.shape != (batch, length):
58
+ raise ValueError(f"mask must be shaped {(batch, length)}, got {tuple(mask.shape)}")
59
+ return mask.to(device=device, dtype=torch.bool)
60
+
61
+
62
+ def build_block_sparse_bool_masks(
63
+ prefix_valid: Optional[torch.Tensor],
64
+ prefix_att: Optional[torch.Tensor],
65
+ *,
66
+ batch: int,
67
+ prefix_len: int,
68
+ action_len: int,
69
+ action_block_size: int,
70
+ non_fast_prefix_len: Optional[int] = None,
71
+ action_valid: Optional[torch.Tensor] = None,
72
+ device: Optional[torch.device] = None,
73
+ ) -> tuple[torch.Tensor, torch.Tensor]:
74
+ """Build boolean masks for the split FlexAttention SDPA calls.
75
+
76
+ Returns ``(prefix_rows, action_rows)`` with shapes ``(B, P, S)`` and
77
+ ``(B, A, S)``. Boolean True means the key/value position is visible.
78
+
79
+ ``prefix_att`` follows Lerobot's cumulative-block convention: prefix key
80
+ ``j`` is visible to prefix query ``i`` when ``cumsum(prefix_att)[j] <=
81
+ cumsum(prefix_att)[i]`` and both rows are valid. When omitted, prefix rows
82
+ attend to all valid prefix tokens.
83
+ """
84
+ if action_block_size <= 0:
85
+ raise ValueError("action_block_size must be positive")
86
+ if prefix_len < 0 or action_len < 0:
87
+ raise ValueError("prefix_len and action_len must be non-negative")
88
+ total_len = prefix_len + action_len
89
+ dev = device
90
+ if dev is None:
91
+ for t in (prefix_valid, prefix_att, action_valid):
92
+ if t is not None:
93
+ dev = t.device
94
+ break
95
+ if dev is None:
96
+ dev = torch.device("cpu")
97
+
98
+ p_valid = _as_valid(prefix_valid, batch, prefix_len, dev)
99
+ a_valid = _as_valid(action_valid, batch, action_len, dev)
100
+
101
+ if prefix_att is None:
102
+ prefix_rows = p_valid[:, :, None] & p_valid[:, None, :]
103
+ else:
104
+ if prefix_att.shape != (batch, prefix_len):
105
+ raise ValueError(
106
+ f"prefix_att must be shaped {(batch, prefix_len)}, got {tuple(prefix_att.shape)}"
107
+ )
108
+ cum = torch.cumsum(prefix_att.to(device=dev, dtype=torch.long), dim=1)
109
+ prefix_rows = (cum[:, None, :] <= cum[:, :, None]) & p_valid[:, :, None] & p_valid[:, None, :]
110
+
111
+ prefix_pad = torch.zeros((batch, prefix_len, action_len), dtype=torch.bool, device=dev)
112
+ prefix_rows = torch.cat([prefix_rows, prefix_pad], dim=2)
113
+
114
+ nf = prefix_len if non_fast_prefix_len is None else int(non_fast_prefix_len)
115
+ nf = max(0, min(nf, prefix_len))
116
+ action_to_prefix = torch.zeros((batch, action_len, prefix_len), dtype=torch.bool, device=dev)
117
+ if nf > 0:
118
+ action_to_prefix[:, :, :nf] = p_valid[:, None, :nf]
119
+ action_to_prefix &= a_valid[:, :, None]
120
+
121
+ q_block = torch.arange(action_len, device=dev) // int(action_block_size)
122
+ kv_block = q_block
123
+ action_block = q_block[:, None] == kv_block[None, :]
124
+ action_block = action_block[None, :, :].expand(batch, -1, -1)
125
+ action_block = action_block & a_valid[:, :, None] & a_valid[:, None, :]
126
+ action_rows = torch.cat([action_to_prefix, action_block], dim=2)
127
+
128
+ if prefix_rows.shape != (batch, prefix_len, total_len):
129
+ raise AssertionError("internal prefix mask shape error")
130
+ if action_rows.shape != (batch, action_len, total_len):
131
+ raise AssertionError("internal action mask shape error")
132
+ return prefix_rows, action_rows
133
+
134
+
135
+ def _bool_to_sdpa_mask(mask: torch.Tensor, q: torch.Tensor) -> torch.Tensor:
136
+ value = MASK_VALUE_F32
137
+ if q.dtype.is_floating_point:
138
+ finfo = torch.finfo(q.dtype)
139
+ value = max(MASK_VALUE_F32, finfo.min)
140
+ return torch.where(
141
+ mask[:, None, :, :],
142
+ torch.zeros((), dtype=q.dtype, device=q.device),
143
+ torch.full((), value, dtype=q.dtype, device=q.device),
144
+ )
145
+
146
+
147
+ def _slice_attention_mask(
148
+ attention_mask: torch.Tensor,
149
+ start: int,
150
+ end: int,
151
+ q: torch.Tensor,
152
+ ) -> torch.Tensor:
153
+ if attention_mask.dim() == 3:
154
+ mask = attention_mask[:, start:end, :]
155
+ if mask.dtype == torch.bool:
156
+ return mask[:, None, :, :]
157
+ return mask[:, None, :, :].to(dtype=q.dtype)
158
+ if attention_mask.dim() == 4:
159
+ mask = attention_mask[:, :, start:end, :]
160
+ return mask if mask.dtype == torch.bool else mask.to(dtype=q.dtype)
161
+ raise ValueError("attention_mask must be (B, S, S) or (B, 1|H, S, S)")
162
+
163
+
164
+ def _sdpa(
165
+ q: torch.Tensor,
166
+ k: torch.Tensor,
167
+ v: torch.Tensor,
168
+ mask: Optional[torch.Tensor],
169
+ *,
170
+ scale: Optional[float],
171
+ dropout_p: float,
172
+ enable_gqa: bool,
173
+ ) -> torch.Tensor:
174
+ kwargs = {"attn_mask": mask, "dropout_p": float(dropout_p), "scale": scale}
175
+ if enable_gqa:
176
+ kwargs["enable_gqa"] = True
177
+ try:
178
+ return F.scaled_dot_product_attention(q, k, v, **kwargs)
179
+ except TypeError:
180
+ kwargs.pop("enable_gqa", None)
181
+ return F.scaled_dot_product_attention(q, k, v, **kwargs)
182
+
183
+
184
+ def reference_flex_attention(
185
+ q: torch.Tensor,
186
+ k: torch.Tensor,
187
+ v: torch.Tensor,
188
+ *,
189
+ prefix_len: int,
190
+ action_block_size: int,
191
+ attention_mask: Optional[torch.Tensor] = None,
192
+ prefix_valid: Optional[torch.Tensor] = None,
193
+ prefix_att: Optional[torch.Tensor] = None,
194
+ non_fast_prefix_len: Optional[int] = None,
195
+ action_valid: Optional[torch.Tensor] = None,
196
+ detach_prefix_kv_for_action: bool = True,
197
+ scale: Optional[float] = None,
198
+ dropout_p: float = 0.0,
199
+ enable_gqa: Optional[bool] = None,
200
+ ) -> torch.Tensor:
201
+ """SDPA reference for the PI052 FlexAttention replacement shape.
202
+
203
+ Args:
204
+ q, k, v: ``(B, Hq/Hkv, S, D)`` tensors.
205
+ prefix_len: number of prefix rows/columns at the start of sequence.
206
+ action_block_size: size of each block-diagonal action segment.
207
+ attention_mask: optional prebuilt additive or boolean full mask.
208
+ prefix_valid: optional ``(B, P)`` valid prefix positions.
209
+ prefix_att: optional ``(B, P)`` cumulative-block markers.
210
+ non_fast_prefix_len: prefix columns visible to action rows.
211
+ action_valid: optional ``(B, A)`` valid action positions.
212
+ detach_prefix_kv_for_action: detach prefix K/V on the action-row path.
213
+ scale: SDPA scale. Defaults to ``D ** -0.5``.
214
+ dropout_p: SDPA dropout probability.
215
+ enable_gqa: pass SDPA GQA mode when q heads and kv heads differ.
216
+ """
217
+ _check_qkv(q, k, v)
218
+ batch, _, total_len, head_dim = q.shape
219
+ if not (0 <= int(prefix_len) <= total_len):
220
+ raise ValueError("prefix_len must be in [0, S]")
221
+ prefix_len = int(prefix_len)
222
+ action_len = total_len - prefix_len
223
+ if scale is None:
224
+ scale = head_dim**-0.5
225
+ if enable_gqa is None:
226
+ enable_gqa = q.shape[1] != k.shape[1]
227
+
228
+ q_prefix = q[:, :, :prefix_len, :]
229
+ q_action = q[:, :, prefix_len:, :]
230
+ k_prefix = k[:, :, :prefix_len, :]
231
+ k_action = k[:, :, prefix_len:, :]
232
+ v_prefix = v[:, :, :prefix_len, :]
233
+ v_action = v[:, :, prefix_len:, :]
234
+
235
+ if attention_mask is None:
236
+ prefix_bool, action_bool = build_block_sparse_bool_masks(
237
+ prefix_valid,
238
+ prefix_att,
239
+ batch=batch,
240
+ prefix_len=prefix_len,
241
+ action_len=action_len,
242
+ action_block_size=action_block_size,
243
+ non_fast_prefix_len=non_fast_prefix_len,
244
+ action_valid=action_valid,
245
+ device=q.device,
246
+ )
247
+ prefix_mask = _bool_to_sdpa_mask(prefix_bool, q)
248
+ action_mask = _bool_to_sdpa_mask(action_bool, q)
249
+ else:
250
+ prefix_mask = _slice_attention_mask(attention_mask, 0, prefix_len, q)
251
+ action_mask = _slice_attention_mask(attention_mask, prefix_len, total_len, q)
252
+
253
+ out_parts = []
254
+ if prefix_len:
255
+ out_parts.append(
256
+ _sdpa(
257
+ q_prefix,
258
+ k,
259
+ v,
260
+ prefix_mask,
261
+ scale=scale,
262
+ dropout_p=dropout_p,
263
+ enable_gqa=bool(enable_gqa),
264
+ )
265
+ )
266
+ if action_len:
267
+ prefix_k = k_prefix.detach() if detach_prefix_kv_for_action else k_prefix
268
+ prefix_v = v_prefix.detach() if detach_prefix_kv_for_action else v_prefix
269
+ k_for_action = torch.cat([prefix_k, k_action], dim=2)
270
+ v_for_action = torch.cat([prefix_v, v_action], dim=2)
271
+ out_parts.append(
272
+ _sdpa(
273
+ q_action,
274
+ k_for_action,
275
+ v_for_action,
276
+ action_mask,
277
+ scale=scale,
278
+ dropout_p=dropout_p,
279
+ enable_gqa=bool(enable_gqa),
280
+ )
281
+ )
282
+ if not out_parts:
283
+ return q.new_empty(q.shape)
284
+ return torch.cat(out_parts, dim=2) if len(out_parts) == 2 else out_parts[0]
285
+
286
+
287
+ def _manual_attention_part(qs, ks, vs, mask, scale):
288
+ """Materialized-logits attention part: cuBLAS GEMMs + fused masked softmax.
289
+
290
+ Same math as SDPA with an additive mask (fp32 softmax; logits stored in
291
+ the io dtype between the GEMM and the softmax). Grouped queries run as a
292
+ strided batched GEMM over the KV heads, so a 1-head K/V is never
293
+ repeated. At PI052 training shapes (GQA 8:1, D=256, bf16) this beats
294
+ both SDPA-with-dense-mask (2.3-3.1x) and the best FlexAttention
295
+ configuration (1.4-2.9x) on fwd+bwd — see benchmarks/RESULTS.md.
296
+ """
297
+ B, H, Sq, D = qs.shape
298
+ Hk = ks.shape[1]
299
+ if Hk != H:
300
+ g = H // Hk
301
+ q2 = qs.reshape(B, Hk, g * Sq, D)
302
+ logits = (q2 @ ks.transpose(-1, -2)).reshape(B, H, Sq, -1)
303
+ else:
304
+ logits = qs @ ks.transpose(-1, -2)
305
+ logits = logits * scale
306
+ if mask is not None:
307
+ logits = logits + mask
308
+ p = logits.float().softmax(dim=-1).to(qs.dtype)
309
+ if Hk != H:
310
+ out = (p.reshape(B, Hk, g * Sq, -1) @ vs).reshape(B, H, Sq, D)
311
+ else:
312
+ out = p @ vs
313
+ return out
314
+
315
+
316
+ _manual_part_compiled = None
317
+
318
+
319
+ def _get_manual_part():
320
+ global _manual_part_compiled
321
+ if _manual_part_compiled is None:
322
+ _manual_part_compiled = torch.compile(_manual_attention_part, dynamic=False)
323
+ return _manual_part_compiled
324
+
325
+
326
+ def manual_attention(
327
+ q: torch.Tensor,
328
+ k: torch.Tensor,
329
+ v: torch.Tensor,
330
+ *,
331
+ prefix_len: int,
332
+ action_block_size: int,
333
+ attention_mask: Optional[torch.Tensor] = None,
334
+ prefix_valid: Optional[torch.Tensor] = None,
335
+ prefix_att: Optional[torch.Tensor] = None,
336
+ non_fast_prefix_len: Optional[int] = None,
337
+ action_valid: Optional[torch.Tensor] = None,
338
+ detach_prefix_kv_for_action: bool = True,
339
+ scale: Optional[float] = None,
340
+ dropout_p: float = 0.0,
341
+ compile_part: bool = True,
342
+ ) -> torch.Tensor:
343
+ """Materialized-logits implementation of :func:`reference_flex_attention`.
344
+
345
+ Same mask semantics and prefix/action split; each part runs through
346
+ :func:`_manual_attention_part` instead of SDPA. ``dropout_p`` must be 0
347
+ (training attention dropout is unused in PI052); other values raise so
348
+ callers fall back explicitly.
349
+ """
350
+ if dropout_p:
351
+ raise ValueError("manual_attention does not support dropout; use the reference path")
352
+ _check_qkv(q, k, v)
353
+ batch, _, total_len, head_dim = q.shape
354
+ if not (0 <= int(prefix_len) <= total_len):
355
+ raise ValueError("prefix_len must be in [0, S]")
356
+ prefix_len = int(prefix_len)
357
+ action_len = total_len - prefix_len
358
+ if scale is None:
359
+ scale = head_dim**-0.5
360
+
361
+ if attention_mask is None:
362
+ prefix_bool, action_bool = build_block_sparse_bool_masks(
363
+ prefix_valid,
364
+ prefix_att,
365
+ batch=batch,
366
+ prefix_len=prefix_len,
367
+ action_len=action_len,
368
+ action_block_size=action_block_size,
369
+ non_fast_prefix_len=non_fast_prefix_len,
370
+ action_valid=action_valid,
371
+ device=q.device,
372
+ )
373
+ prefix_mask = _bool_to_sdpa_mask(prefix_bool, q)
374
+ action_mask = _bool_to_sdpa_mask(action_bool, q)
375
+ else:
376
+ prefix_mask = _slice_attention_mask(attention_mask, 0, prefix_len, q)
377
+ action_mask = _slice_attention_mask(attention_mask, prefix_len, total_len, q)
378
+ if prefix_mask.dtype == torch.bool:
379
+ prefix_mask = _bool_to_sdpa_mask(prefix_mask[:, 0], q)
380
+ if action_mask.dtype == torch.bool:
381
+ action_mask = _bool_to_sdpa_mask(action_mask[:, 0], q)
382
+
383
+ part = _get_manual_part() if compile_part else _manual_attention_part
384
+ out_parts = []
385
+ if prefix_len:
386
+ out_parts.append(part(q[:, :, :prefix_len, :], k, v, prefix_mask, scale))
387
+ if action_len:
388
+ k_prefix = k[:, :, :prefix_len, :]
389
+ v_prefix = v[:, :, :prefix_len, :]
390
+ if detach_prefix_kv_for_action:
391
+ k_prefix = k_prefix.detach()
392
+ v_prefix = v_prefix.detach()
393
+ k_for_action = torch.cat([k_prefix, k[:, :, prefix_len:, :]], dim=2)
394
+ v_for_action = torch.cat([v_prefix, v[:, :, prefix_len:, :]], dim=2)
395
+ out_parts.append(part(q[:, :, prefix_len:, :], k_for_action, v_for_action, action_mask, scale))
396
+ if not out_parts:
397
+ return q.new_empty(q.shape)
398
+ return torch.cat(out_parts, dim=2) if len(out_parts) == 2 else out_parts[0]
399
+
400
+
401
+ def flex_attention(
402
+ q: torch.Tensor,
403
+ k: torch.Tensor,
404
+ v: torch.Tensor,
405
+ *,
406
+ prefix_len: int,
407
+ action_block_size: int,
408
+ attention_mask: Optional[torch.Tensor] = None,
409
+ prefix_valid: Optional[torch.Tensor] = None,
410
+ prefix_att: Optional[torch.Tensor] = None,
411
+ non_fast_prefix_len: Optional[int] = None,
412
+ action_valid: Optional[torch.Tensor] = None,
413
+ detach_prefix_kv_for_action: bool = True,
414
+ scale: Optional[float] = None,
415
+ dropout_p: float = 0.0,
416
+ enable_gqa: Optional[bool] = None,
417
+ force_fallback: bool = False,
418
+ impl: str = "sdpa",
419
+ ) -> torch.Tensor:
420
+ """Flex-style block-sparse attention.
421
+
422
+ ``impl="sdpa"`` (default) keeps the SDPA reference path;
423
+ ``impl="manual"`` routes through the materialized-logits
424
+ implementation; ``impl="auto"`` picks manual only where it has been
425
+ measured to win end-to-end — consumer Blackwell (sm120-class) with
426
+ no dropout. On A100 (sm80) the manual math wins microbenches but
427
+ loses training-step integration, and on H100/H200 (sm90) the fused
428
+ FMHA kernels win outright, so auto keeps SDPA there.
429
+ """
430
+ _ = force_fallback
431
+ if impl == "auto":
432
+ sm120 = q.is_cuda and torch.cuda.get_device_capability(q.device)[0] == 12
433
+ impl = "manual" if (sm120 and not dropout_p) else "sdpa"
434
+ if impl == "manual":
435
+ return manual_attention(
436
+ q,
437
+ k,
438
+ v,
439
+ prefix_len=prefix_len,
440
+ action_block_size=action_block_size,
441
+ attention_mask=attention_mask,
442
+ prefix_valid=prefix_valid,
443
+ prefix_att=prefix_att,
444
+ non_fast_prefix_len=non_fast_prefix_len,
445
+ action_valid=action_valid,
446
+ detach_prefix_kv_for_action=detach_prefix_kv_for_action,
447
+ scale=scale,
448
+ dropout_p=dropout_p,
449
+ )
450
+ return reference_flex_attention(
451
+ q,
452
+ k,
453
+ v,
454
+ prefix_len=prefix_len,
455
+ action_block_size=action_block_size,
456
+ attention_mask=attention_mask,
457
+ prefix_valid=prefix_valid,
458
+ prefix_att=prefix_att,
459
+ non_fast_prefix_len=non_fast_prefix_len,
460
+ action_valid=action_valid,
461
+ detach_prefix_kv_for_action=detach_prefix_kv_for_action,
462
+ scale=scale,
463
+ dropout_p=dropout_p,
464
+ enable_gqa=enable_gqa,
465
+ )
466
+
467
+
468
+ def flex_attention_forward(*args, **kwargs) -> torch.Tensor:
469
+ """Forward-only compatibility wrapper."""
470
+ return flex_attention(*args, **kwargs)
471
+
472
+
473
+ def backend_marker(x: torch.Tensor) -> torch.Tensor:
474
+ if ops is None:
475
+ return x
476
+ return ops._flashrt_training_package_marker(x)
477
+
478
+
479
+ __all__ = [
480
+ "MASK_VALUE_F32",
481
+ "backend_marker",
482
+ "build_block_sparse_bool_masks",
483
+ "flex_attention",
484
+ "flex_attention_forward",
485
+ "manual_attention",
486
+ "reference_flex_attention",
487
+ ]
build/torch212-cxx11-cu130-x86_64-linux/_flashrt_flex_attention_train_cuda_eaf7454.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6afaa6ed896df3e147d469f64c9c08fde09d265c3c6a8a07489f97abfdbb5b77
3
+ size 56800
build/torch212-cxx11-cu130-x86_64-linux/_ops.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _flashrt_flex_attention_train_cuda_eaf7454
3
+ ops = torch.ops._flashrt_flex_attention_train_cuda_eaf7454
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ """
7
+ Prefix op by namespace.
8
+ """
9
+ return f"_flashrt_flex_attention_train_cuda_eaf7454::{op_name}"
build/torch212-cxx11-cu130-x86_64-linux/flashrt_flex_attention_train/__init__.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import ctypes
2
+ import importlib.util
3
+ import sys
4
+ from pathlib import Path
5
+ from types import ModuleType
6
+
7
+
8
+ def _import_from_path(file_path: Path) -> ModuleType:
9
+ # We cannot use the module name as-is, after adding it to `sys.modules`,
10
+ # it would also be used for other imports. So, we make a module name that
11
+ # depends on the path for it to be unique using the hex-encoded hash of
12
+ # the path.
13
+ path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
14
+ module_name = path_hash
15
+ spec = importlib.util.spec_from_file_location(module_name, file_path)
16
+ if spec is None:
17
+ raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
18
+ module = importlib.util.module_from_spec(spec)
19
+ if module is None:
20
+ raise ImportError(f"Cannot load module {module_name} from spec")
21
+ sys.modules[module_name] = module
22
+ spec.loader.exec_module(module) # type: ignore
23
+ return module
24
+
25
+
26
+ globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
build/torch212-cxx11-cu130-x86_64-linux/metadata.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "flashrt-flex-attention-train",
3
+ "id": "_flashrt_flex_attention_train_cuda_eaf7454",
4
+ "version": 1,
5
+ "license": "Apache-2.0",
6
+ "python-depends": [],
7
+ "backend": {
8
+ "type": "cuda",
9
+ "archs": [
10
+ "10.0",
11
+ "11.0",
12
+ "12.0",
13
+ "12.1+PTX",
14
+ "7.5",
15
+ "8.0",
16
+ "8.6",
17
+ "8.7",
18
+ "8.9",
19
+ "9.0"
20
+ ]
21
+ }
22
+ }
build/torch212-cxx11-cu132-x86_64-linux/__init__.py ADDED
@@ -0,0 +1,487 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FlashRT Flex-style block-sparse attention training API.
2
+
3
+ The public function implements the PI052 prefix/action mask pattern:
4
+
5
+ * prefix query rows use the original K/V tensors, so prefix losses keep normal
6
+ gradients into prefix K/V;
7
+ * action query rows read detached prefix K/V plus normal action K/V by default,
8
+ matching the current training semantics.
9
+
10
+ Unsupported shapes route to the SDPA reference path. Native CUDA kernels are
11
+ not exposed until a shape-specialized implementation beats SDPA on the target
12
+ A100/5090 validation matrix.
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ from typing import Optional
18
+
19
+ import torch
20
+ import torch.nn.functional as F
21
+
22
+ try:
23
+ from ._ops import ops
24
+
25
+ _HAS_OPS = hasattr(ops, "_flashrt_training_package_marker")
26
+ except Exception: # source-tree tests before kernel-builder creates _ops.py
27
+ ops = None
28
+ _HAS_OPS = False
29
+
30
+
31
+ MASK_VALUE_F32 = -2.3819763e38
32
+
33
+
34
+ def _use_ops(namespace_ops) -> None:
35
+ """Install a manually built extension (dev/testing path)."""
36
+ global ops, _HAS_OPS
37
+ ops = namespace_ops
38
+ _HAS_OPS = hasattr(ops, "_flashrt_training_package_marker")
39
+
40
+
41
+ def _check_qkv(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor) -> None:
42
+ if q.dim() != 4 or k.dim() != 4 or v.dim() != 4:
43
+ raise ValueError("q, k, and v must be shaped (B, H, S, D)")
44
+ if q.shape[0] != k.shape[0] or q.shape[0] != v.shape[0]:
45
+ raise ValueError("q, k, and v batch dimensions must match")
46
+ if k.shape != v.shape:
47
+ raise ValueError("k and v shapes must match")
48
+ if q.shape[2] != k.shape[2] or q.shape[3] != k.shape[3]:
49
+ raise ValueError("q, k, and v sequence/head_dim dimensions must match")
50
+ if q.device != k.device or q.device != v.device:
51
+ raise ValueError("q, k, and v must be on the same device")
52
+
53
+
54
+ def _as_valid(mask: Optional[torch.Tensor], batch: int, length: int, device: torch.device) -> torch.Tensor:
55
+ if mask is None:
56
+ return torch.ones((batch, length), dtype=torch.bool, device=device)
57
+ if mask.shape != (batch, length):
58
+ raise ValueError(f"mask must be shaped {(batch, length)}, got {tuple(mask.shape)}")
59
+ return mask.to(device=device, dtype=torch.bool)
60
+
61
+
62
+ def build_block_sparse_bool_masks(
63
+ prefix_valid: Optional[torch.Tensor],
64
+ prefix_att: Optional[torch.Tensor],
65
+ *,
66
+ batch: int,
67
+ prefix_len: int,
68
+ action_len: int,
69
+ action_block_size: int,
70
+ non_fast_prefix_len: Optional[int] = None,
71
+ action_valid: Optional[torch.Tensor] = None,
72
+ device: Optional[torch.device] = None,
73
+ ) -> tuple[torch.Tensor, torch.Tensor]:
74
+ """Build boolean masks for the split FlexAttention SDPA calls.
75
+
76
+ Returns ``(prefix_rows, action_rows)`` with shapes ``(B, P, S)`` and
77
+ ``(B, A, S)``. Boolean True means the key/value position is visible.
78
+
79
+ ``prefix_att`` follows Lerobot's cumulative-block convention: prefix key
80
+ ``j`` is visible to prefix query ``i`` when ``cumsum(prefix_att)[j] <=
81
+ cumsum(prefix_att)[i]`` and both rows are valid. When omitted, prefix rows
82
+ attend to all valid prefix tokens.
83
+ """
84
+ if action_block_size <= 0:
85
+ raise ValueError("action_block_size must be positive")
86
+ if prefix_len < 0 or action_len < 0:
87
+ raise ValueError("prefix_len and action_len must be non-negative")
88
+ total_len = prefix_len + action_len
89
+ dev = device
90
+ if dev is None:
91
+ for t in (prefix_valid, prefix_att, action_valid):
92
+ if t is not None:
93
+ dev = t.device
94
+ break
95
+ if dev is None:
96
+ dev = torch.device("cpu")
97
+
98
+ p_valid = _as_valid(prefix_valid, batch, prefix_len, dev)
99
+ a_valid = _as_valid(action_valid, batch, action_len, dev)
100
+
101
+ if prefix_att is None:
102
+ prefix_rows = p_valid[:, :, None] & p_valid[:, None, :]
103
+ else:
104
+ if prefix_att.shape != (batch, prefix_len):
105
+ raise ValueError(
106
+ f"prefix_att must be shaped {(batch, prefix_len)}, got {tuple(prefix_att.shape)}"
107
+ )
108
+ cum = torch.cumsum(prefix_att.to(device=dev, dtype=torch.long), dim=1)
109
+ prefix_rows = (cum[:, None, :] <= cum[:, :, None]) & p_valid[:, :, None] & p_valid[:, None, :]
110
+
111
+ prefix_pad = torch.zeros((batch, prefix_len, action_len), dtype=torch.bool, device=dev)
112
+ prefix_rows = torch.cat([prefix_rows, prefix_pad], dim=2)
113
+
114
+ nf = prefix_len if non_fast_prefix_len is None else int(non_fast_prefix_len)
115
+ nf = max(0, min(nf, prefix_len))
116
+ action_to_prefix = torch.zeros((batch, action_len, prefix_len), dtype=torch.bool, device=dev)
117
+ if nf > 0:
118
+ action_to_prefix[:, :, :nf] = p_valid[:, None, :nf]
119
+ action_to_prefix &= a_valid[:, :, None]
120
+
121
+ q_block = torch.arange(action_len, device=dev) // int(action_block_size)
122
+ kv_block = q_block
123
+ action_block = q_block[:, None] == kv_block[None, :]
124
+ action_block = action_block[None, :, :].expand(batch, -1, -1)
125
+ action_block = action_block & a_valid[:, :, None] & a_valid[:, None, :]
126
+ action_rows = torch.cat([action_to_prefix, action_block], dim=2)
127
+
128
+ if prefix_rows.shape != (batch, prefix_len, total_len):
129
+ raise AssertionError("internal prefix mask shape error")
130
+ if action_rows.shape != (batch, action_len, total_len):
131
+ raise AssertionError("internal action mask shape error")
132
+ return prefix_rows, action_rows
133
+
134
+
135
+ def _bool_to_sdpa_mask(mask: torch.Tensor, q: torch.Tensor) -> torch.Tensor:
136
+ value = MASK_VALUE_F32
137
+ if q.dtype.is_floating_point:
138
+ finfo = torch.finfo(q.dtype)
139
+ value = max(MASK_VALUE_F32, finfo.min)
140
+ return torch.where(
141
+ mask[:, None, :, :],
142
+ torch.zeros((), dtype=q.dtype, device=q.device),
143
+ torch.full((), value, dtype=q.dtype, device=q.device),
144
+ )
145
+
146
+
147
+ def _slice_attention_mask(
148
+ attention_mask: torch.Tensor,
149
+ start: int,
150
+ end: int,
151
+ q: torch.Tensor,
152
+ ) -> torch.Tensor:
153
+ if attention_mask.dim() == 3:
154
+ mask = attention_mask[:, start:end, :]
155
+ if mask.dtype == torch.bool:
156
+ return mask[:, None, :, :]
157
+ return mask[:, None, :, :].to(dtype=q.dtype)
158
+ if attention_mask.dim() == 4:
159
+ mask = attention_mask[:, :, start:end, :]
160
+ return mask if mask.dtype == torch.bool else mask.to(dtype=q.dtype)
161
+ raise ValueError("attention_mask must be (B, S, S) or (B, 1|H, S, S)")
162
+
163
+
164
+ def _sdpa(
165
+ q: torch.Tensor,
166
+ k: torch.Tensor,
167
+ v: torch.Tensor,
168
+ mask: Optional[torch.Tensor],
169
+ *,
170
+ scale: Optional[float],
171
+ dropout_p: float,
172
+ enable_gqa: bool,
173
+ ) -> torch.Tensor:
174
+ kwargs = {"attn_mask": mask, "dropout_p": float(dropout_p), "scale": scale}
175
+ if enable_gqa:
176
+ kwargs["enable_gqa"] = True
177
+ try:
178
+ return F.scaled_dot_product_attention(q, k, v, **kwargs)
179
+ except TypeError:
180
+ kwargs.pop("enable_gqa", None)
181
+ return F.scaled_dot_product_attention(q, k, v, **kwargs)
182
+
183
+
184
+ def reference_flex_attention(
185
+ q: torch.Tensor,
186
+ k: torch.Tensor,
187
+ v: torch.Tensor,
188
+ *,
189
+ prefix_len: int,
190
+ action_block_size: int,
191
+ attention_mask: Optional[torch.Tensor] = None,
192
+ prefix_valid: Optional[torch.Tensor] = None,
193
+ prefix_att: Optional[torch.Tensor] = None,
194
+ non_fast_prefix_len: Optional[int] = None,
195
+ action_valid: Optional[torch.Tensor] = None,
196
+ detach_prefix_kv_for_action: bool = True,
197
+ scale: Optional[float] = None,
198
+ dropout_p: float = 0.0,
199
+ enable_gqa: Optional[bool] = None,
200
+ ) -> torch.Tensor:
201
+ """SDPA reference for the PI052 FlexAttention replacement shape.
202
+
203
+ Args:
204
+ q, k, v: ``(B, Hq/Hkv, S, D)`` tensors.
205
+ prefix_len: number of prefix rows/columns at the start of sequence.
206
+ action_block_size: size of each block-diagonal action segment.
207
+ attention_mask: optional prebuilt additive or boolean full mask.
208
+ prefix_valid: optional ``(B, P)`` valid prefix positions.
209
+ prefix_att: optional ``(B, P)`` cumulative-block markers.
210
+ non_fast_prefix_len: prefix columns visible to action rows.
211
+ action_valid: optional ``(B, A)`` valid action positions.
212
+ detach_prefix_kv_for_action: detach prefix K/V on the action-row path.
213
+ scale: SDPA scale. Defaults to ``D ** -0.5``.
214
+ dropout_p: SDPA dropout probability.
215
+ enable_gqa: pass SDPA GQA mode when q heads and kv heads differ.
216
+ """
217
+ _check_qkv(q, k, v)
218
+ batch, _, total_len, head_dim = q.shape
219
+ if not (0 <= int(prefix_len) <= total_len):
220
+ raise ValueError("prefix_len must be in [0, S]")
221
+ prefix_len = int(prefix_len)
222
+ action_len = total_len - prefix_len
223
+ if scale is None:
224
+ scale = head_dim**-0.5
225
+ if enable_gqa is None:
226
+ enable_gqa = q.shape[1] != k.shape[1]
227
+
228
+ q_prefix = q[:, :, :prefix_len, :]
229
+ q_action = q[:, :, prefix_len:, :]
230
+ k_prefix = k[:, :, :prefix_len, :]
231
+ k_action = k[:, :, prefix_len:, :]
232
+ v_prefix = v[:, :, :prefix_len, :]
233
+ v_action = v[:, :, prefix_len:, :]
234
+
235
+ if attention_mask is None:
236
+ prefix_bool, action_bool = build_block_sparse_bool_masks(
237
+ prefix_valid,
238
+ prefix_att,
239
+ batch=batch,
240
+ prefix_len=prefix_len,
241
+ action_len=action_len,
242
+ action_block_size=action_block_size,
243
+ non_fast_prefix_len=non_fast_prefix_len,
244
+ action_valid=action_valid,
245
+ device=q.device,
246
+ )
247
+ prefix_mask = _bool_to_sdpa_mask(prefix_bool, q)
248
+ action_mask = _bool_to_sdpa_mask(action_bool, q)
249
+ else:
250
+ prefix_mask = _slice_attention_mask(attention_mask, 0, prefix_len, q)
251
+ action_mask = _slice_attention_mask(attention_mask, prefix_len, total_len, q)
252
+
253
+ out_parts = []
254
+ if prefix_len:
255
+ out_parts.append(
256
+ _sdpa(
257
+ q_prefix,
258
+ k,
259
+ v,
260
+ prefix_mask,
261
+ scale=scale,
262
+ dropout_p=dropout_p,
263
+ enable_gqa=bool(enable_gqa),
264
+ )
265
+ )
266
+ if action_len:
267
+ prefix_k = k_prefix.detach() if detach_prefix_kv_for_action else k_prefix
268
+ prefix_v = v_prefix.detach() if detach_prefix_kv_for_action else v_prefix
269
+ k_for_action = torch.cat([prefix_k, k_action], dim=2)
270
+ v_for_action = torch.cat([prefix_v, v_action], dim=2)
271
+ out_parts.append(
272
+ _sdpa(
273
+ q_action,
274
+ k_for_action,
275
+ v_for_action,
276
+ action_mask,
277
+ scale=scale,
278
+ dropout_p=dropout_p,
279
+ enable_gqa=bool(enable_gqa),
280
+ )
281
+ )
282
+ if not out_parts:
283
+ return q.new_empty(q.shape)
284
+ return torch.cat(out_parts, dim=2) if len(out_parts) == 2 else out_parts[0]
285
+
286
+
287
+ def _manual_attention_part(qs, ks, vs, mask, scale):
288
+ """Materialized-logits attention part: cuBLAS GEMMs + fused masked softmax.
289
+
290
+ Same math as SDPA with an additive mask (fp32 softmax; logits stored in
291
+ the io dtype between the GEMM and the softmax). Grouped queries run as a
292
+ strided batched GEMM over the KV heads, so a 1-head K/V is never
293
+ repeated. At PI052 training shapes (GQA 8:1, D=256, bf16) this beats
294
+ both SDPA-with-dense-mask (2.3-3.1x) and the best FlexAttention
295
+ configuration (1.4-2.9x) on fwd+bwd — see benchmarks/RESULTS.md.
296
+ """
297
+ B, H, Sq, D = qs.shape
298
+ Hk = ks.shape[1]
299
+ if Hk != H:
300
+ g = H // Hk
301
+ q2 = qs.reshape(B, Hk, g * Sq, D)
302
+ logits = (q2 @ ks.transpose(-1, -2)).reshape(B, H, Sq, -1)
303
+ else:
304
+ logits = qs @ ks.transpose(-1, -2)
305
+ logits = logits * scale
306
+ if mask is not None:
307
+ logits = logits + mask
308
+ p = logits.float().softmax(dim=-1).to(qs.dtype)
309
+ if Hk != H:
310
+ out = (p.reshape(B, Hk, g * Sq, -1) @ vs).reshape(B, H, Sq, D)
311
+ else:
312
+ out = p @ vs
313
+ return out
314
+
315
+
316
+ _manual_part_compiled = None
317
+
318
+
319
+ def _get_manual_part():
320
+ global _manual_part_compiled
321
+ if _manual_part_compiled is None:
322
+ _manual_part_compiled = torch.compile(_manual_attention_part, dynamic=False)
323
+ return _manual_part_compiled
324
+
325
+
326
+ def manual_attention(
327
+ q: torch.Tensor,
328
+ k: torch.Tensor,
329
+ v: torch.Tensor,
330
+ *,
331
+ prefix_len: int,
332
+ action_block_size: int,
333
+ attention_mask: Optional[torch.Tensor] = None,
334
+ prefix_valid: Optional[torch.Tensor] = None,
335
+ prefix_att: Optional[torch.Tensor] = None,
336
+ non_fast_prefix_len: Optional[int] = None,
337
+ action_valid: Optional[torch.Tensor] = None,
338
+ detach_prefix_kv_for_action: bool = True,
339
+ scale: Optional[float] = None,
340
+ dropout_p: float = 0.0,
341
+ compile_part: bool = True,
342
+ ) -> torch.Tensor:
343
+ """Materialized-logits implementation of :func:`reference_flex_attention`.
344
+
345
+ Same mask semantics and prefix/action split; each part runs through
346
+ :func:`_manual_attention_part` instead of SDPA. ``dropout_p`` must be 0
347
+ (training attention dropout is unused in PI052); other values raise so
348
+ callers fall back explicitly.
349
+ """
350
+ if dropout_p:
351
+ raise ValueError("manual_attention does not support dropout; use the reference path")
352
+ _check_qkv(q, k, v)
353
+ batch, _, total_len, head_dim = q.shape
354
+ if not (0 <= int(prefix_len) <= total_len):
355
+ raise ValueError("prefix_len must be in [0, S]")
356
+ prefix_len = int(prefix_len)
357
+ action_len = total_len - prefix_len
358
+ if scale is None:
359
+ scale = head_dim**-0.5
360
+
361
+ if attention_mask is None:
362
+ prefix_bool, action_bool = build_block_sparse_bool_masks(
363
+ prefix_valid,
364
+ prefix_att,
365
+ batch=batch,
366
+ prefix_len=prefix_len,
367
+ action_len=action_len,
368
+ action_block_size=action_block_size,
369
+ non_fast_prefix_len=non_fast_prefix_len,
370
+ action_valid=action_valid,
371
+ device=q.device,
372
+ )
373
+ prefix_mask = _bool_to_sdpa_mask(prefix_bool, q)
374
+ action_mask = _bool_to_sdpa_mask(action_bool, q)
375
+ else:
376
+ prefix_mask = _slice_attention_mask(attention_mask, 0, prefix_len, q)
377
+ action_mask = _slice_attention_mask(attention_mask, prefix_len, total_len, q)
378
+ if prefix_mask.dtype == torch.bool:
379
+ prefix_mask = _bool_to_sdpa_mask(prefix_mask[:, 0], q)
380
+ if action_mask.dtype == torch.bool:
381
+ action_mask = _bool_to_sdpa_mask(action_mask[:, 0], q)
382
+
383
+ part = _get_manual_part() if compile_part else _manual_attention_part
384
+ out_parts = []
385
+ if prefix_len:
386
+ out_parts.append(part(q[:, :, :prefix_len, :], k, v, prefix_mask, scale))
387
+ if action_len:
388
+ k_prefix = k[:, :, :prefix_len, :]
389
+ v_prefix = v[:, :, :prefix_len, :]
390
+ if detach_prefix_kv_for_action:
391
+ k_prefix = k_prefix.detach()
392
+ v_prefix = v_prefix.detach()
393
+ k_for_action = torch.cat([k_prefix, k[:, :, prefix_len:, :]], dim=2)
394
+ v_for_action = torch.cat([v_prefix, v[:, :, prefix_len:, :]], dim=2)
395
+ out_parts.append(part(q[:, :, prefix_len:, :], k_for_action, v_for_action, action_mask, scale))
396
+ if not out_parts:
397
+ return q.new_empty(q.shape)
398
+ return torch.cat(out_parts, dim=2) if len(out_parts) == 2 else out_parts[0]
399
+
400
+
401
+ def flex_attention(
402
+ q: torch.Tensor,
403
+ k: torch.Tensor,
404
+ v: torch.Tensor,
405
+ *,
406
+ prefix_len: int,
407
+ action_block_size: int,
408
+ attention_mask: Optional[torch.Tensor] = None,
409
+ prefix_valid: Optional[torch.Tensor] = None,
410
+ prefix_att: Optional[torch.Tensor] = None,
411
+ non_fast_prefix_len: Optional[int] = None,
412
+ action_valid: Optional[torch.Tensor] = None,
413
+ detach_prefix_kv_for_action: bool = True,
414
+ scale: Optional[float] = None,
415
+ dropout_p: float = 0.0,
416
+ enable_gqa: Optional[bool] = None,
417
+ force_fallback: bool = False,
418
+ impl: str = "sdpa",
419
+ ) -> torch.Tensor:
420
+ """Flex-style block-sparse attention.
421
+
422
+ ``impl="sdpa"`` (default) keeps the SDPA reference path;
423
+ ``impl="manual"`` routes through the materialized-logits
424
+ implementation; ``impl="auto"`` picks manual only where it has been
425
+ measured to win end-to-end — consumer Blackwell (sm120-class) with
426
+ no dropout. On A100 (sm80) the manual math wins microbenches but
427
+ loses training-step integration, and on H100/H200 (sm90) the fused
428
+ FMHA kernels win outright, so auto keeps SDPA there.
429
+ """
430
+ _ = force_fallback
431
+ if impl == "auto":
432
+ sm120 = q.is_cuda and torch.cuda.get_device_capability(q.device)[0] == 12
433
+ impl = "manual" if (sm120 and not dropout_p) else "sdpa"
434
+ if impl == "manual":
435
+ return manual_attention(
436
+ q,
437
+ k,
438
+ v,
439
+ prefix_len=prefix_len,
440
+ action_block_size=action_block_size,
441
+ attention_mask=attention_mask,
442
+ prefix_valid=prefix_valid,
443
+ prefix_att=prefix_att,
444
+ non_fast_prefix_len=non_fast_prefix_len,
445
+ action_valid=action_valid,
446
+ detach_prefix_kv_for_action=detach_prefix_kv_for_action,
447
+ scale=scale,
448
+ dropout_p=dropout_p,
449
+ )
450
+ return reference_flex_attention(
451
+ q,
452
+ k,
453
+ v,
454
+ prefix_len=prefix_len,
455
+ action_block_size=action_block_size,
456
+ attention_mask=attention_mask,
457
+ prefix_valid=prefix_valid,
458
+ prefix_att=prefix_att,
459
+ non_fast_prefix_len=non_fast_prefix_len,
460
+ action_valid=action_valid,
461
+ detach_prefix_kv_for_action=detach_prefix_kv_for_action,
462
+ scale=scale,
463
+ dropout_p=dropout_p,
464
+ enable_gqa=enable_gqa,
465
+ )
466
+
467
+
468
+ def flex_attention_forward(*args, **kwargs) -> torch.Tensor:
469
+ """Forward-only compatibility wrapper."""
470
+ return flex_attention(*args, **kwargs)
471
+
472
+
473
+ def backend_marker(x: torch.Tensor) -> torch.Tensor:
474
+ if ops is None:
475
+ return x
476
+ return ops._flashrt_training_package_marker(x)
477
+
478
+
479
+ __all__ = [
480
+ "MASK_VALUE_F32",
481
+ "backend_marker",
482
+ "build_block_sparse_bool_masks",
483
+ "flex_attention",
484
+ "flex_attention_forward",
485
+ "manual_attention",
486
+ "reference_flex_attention",
487
+ ]
build/torch212-cxx11-cu132-x86_64-linux/_flashrt_flex_attention_train_cuda_eaf7454.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:57dd5d0475cf4953b7f770824ce720676641df6cd59ee9b95aef3564a8fb2435
3
+ size 56800
build/torch212-cxx11-cu132-x86_64-linux/_ops.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _flashrt_flex_attention_train_cuda_eaf7454
3
+ ops = torch.ops._flashrt_flex_attention_train_cuda_eaf7454
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ """
7
+ Prefix op by namespace.
8
+ """
9
+ return f"_flashrt_flex_attention_train_cuda_eaf7454::{op_name}"
build/torch212-cxx11-cu132-x86_64-linux/flashrt_flex_attention_train/__init__.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import ctypes
2
+ import importlib.util
3
+ import sys
4
+ from pathlib import Path
5
+ from types import ModuleType
6
+
7
+
8
+ def _import_from_path(file_path: Path) -> ModuleType:
9
+ # We cannot use the module name as-is, after adding it to `sys.modules`,
10
+ # it would also be used for other imports. So, we make a module name that
11
+ # depends on the path for it to be unique using the hex-encoded hash of
12
+ # the path.
13
+ path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
14
+ module_name = path_hash
15
+ spec = importlib.util.spec_from_file_location(module_name, file_path)
16
+ if spec is None:
17
+ raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
18
+ module = importlib.util.module_from_spec(spec)
19
+ if module is None:
20
+ raise ImportError(f"Cannot load module {module_name} from spec")
21
+ sys.modules[module_name] = module
22
+ spec.loader.exec_module(module) # type: ignore
23
+ return module
24
+
25
+
26
+ globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
build/torch212-cxx11-cu132-x86_64-linux/metadata.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "flashrt-flex-attention-train",
3
+ "id": "_flashrt_flex_attention_train_cuda_eaf7454",
4
+ "version": 1,
5
+ "license": "Apache-2.0",
6
+ "python-depends": [],
7
+ "backend": {
8
+ "type": "cuda",
9
+ "archs": [
10
+ "10.0",
11
+ "11.0",
12
+ "12.0",
13
+ "12.1+PTX",
14
+ "7.5",
15
+ "8.0",
16
+ "8.6",
17
+ "8.7",
18
+ "8.9",
19
+ "9.0"
20
+ ]
21
+ }
22
+ }