Uploaded using `kernel-builder`.
Browse files- benchmarks/benchmark.py +779 -0
- build/torch211-cxx11-cu128-x86_64-linux/__init__.py +487 -0
- build/torch211-cxx11-cu128-x86_64-linux/_flashrt_flex_attention_train_cuda_eaf7454.abi3.so +3 -0
- build/torch211-cxx11-cu128-x86_64-linux/_ops.py +9 -0
- build/torch211-cxx11-cu128-x86_64-linux/flashrt_flex_attention_train/__init__.py +26 -0
- build/torch211-cxx11-cu128-x86_64-linux/metadata.json +23 -0
- build/torch211-cxx11-cu130-x86_64-linux/__init__.py +487 -0
- build/torch211-cxx11-cu130-x86_64-linux/_flashrt_flex_attention_train_cuda_eaf7454.abi3.so +3 -0
- build/torch211-cxx11-cu130-x86_64-linux/_ops.py +9 -0
- build/torch211-cxx11-cu130-x86_64-linux/flashrt_flex_attention_train/__init__.py +26 -0
- build/torch211-cxx11-cu130-x86_64-linux/metadata.json +22 -0
- build/torch212-cxx11-cu130-x86_64-linux/__init__.py +487 -0
- build/torch212-cxx11-cu130-x86_64-linux/_flashrt_flex_attention_train_cuda_eaf7454.abi3.so +3 -0
- build/torch212-cxx11-cu130-x86_64-linux/_ops.py +9 -0
- build/torch212-cxx11-cu130-x86_64-linux/flashrt_flex_attention_train/__init__.py +26 -0
- build/torch212-cxx11-cu130-x86_64-linux/metadata.json +22 -0
- build/torch212-cxx11-cu132-x86_64-linux/__init__.py +487 -0
- build/torch212-cxx11-cu132-x86_64-linux/_flashrt_flex_attention_train_cuda_eaf7454.abi3.so +3 -0
- build/torch212-cxx11-cu132-x86_64-linux/_ops.py +9 -0
- build/torch212-cxx11-cu132-x86_64-linux/flashrt_flex_attention_train/__init__.py +26 -0
- build/torch212-cxx11-cu132-x86_64-linux/metadata.json +22 -0
benchmarks/benchmark.py
ADDED
|
@@ -0,0 +1,779 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
import argparse
|
| 5 |
+
import json
|
| 6 |
+
import sys
|
| 7 |
+
import time
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
|
| 10 |
+
import torch
|
| 11 |
+
import torch.nn.functional as F
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
ROOT = Path(__file__).resolve().parents[1]
|
| 15 |
+
sys.path.insert(0, str(ROOT / "torch-ext"))
|
| 16 |
+
import flashrt_flex_attention_train as flex_ops # noqa: E402
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
FLEX_TILE_PRESETS = {
|
| 20 |
+
"default": None,
|
| 21 |
+
# fwd fully autotuned; backward constrained to the consumer-GPU tiles
|
| 22 |
+
# (autotuned backward at GQA/D=256 exceeds 5090 shared memory).
|
| 23 |
+
"bwd_shrunk_only": {
|
| 24 |
+
"bwd_BLOCK_M1": 32,
|
| 25 |
+
"bwd_BLOCK_N1": 64,
|
| 26 |
+
"bwd_BLOCK_M2": 64,
|
| 27 |
+
"bwd_BLOCK_N2": 32,
|
| 28 |
+
},
|
| 29 |
+
"torch_default_explicit": {
|
| 30 |
+
"fwd_BLOCK_M": 32,
|
| 31 |
+
"fwd_BLOCK_N": 64,
|
| 32 |
+
"fwd_num_stages": 2,
|
| 33 |
+
"fwd_num_warps": 4,
|
| 34 |
+
"bwd_BLOCK_M1": 32,
|
| 35 |
+
"bwd_BLOCK_N1": 64,
|
| 36 |
+
"bwd_BLOCK_M2": 64,
|
| 37 |
+
"bwd_BLOCK_N2": 32,
|
| 38 |
+
"bwd_num_stages": 1,
|
| 39 |
+
"bwd_num_warps": 4,
|
| 40 |
+
},
|
| 41 |
+
"a100_d256_bwd_32x64": {
|
| 42 |
+
"fwd_BLOCK_M": 32,
|
| 43 |
+
"fwd_BLOCK_N": 64,
|
| 44 |
+
"fwd_num_stages": 2,
|
| 45 |
+
"fwd_num_warps": 4,
|
| 46 |
+
"bwd_BLOCK_M1": 32,
|
| 47 |
+
"bwd_BLOCK_N1": 64,
|
| 48 |
+
"bwd_BLOCK_M2": 64,
|
| 49 |
+
"bwd_BLOCK_N2": 32,
|
| 50 |
+
"bwd_num_stages": 3,
|
| 51 |
+
"bwd_num_warps": 4,
|
| 52 |
+
},
|
| 53 |
+
"a100_d256_bwd_32x128": {
|
| 54 |
+
"fwd_BLOCK_M": 32,
|
| 55 |
+
"fwd_BLOCK_N": 64,
|
| 56 |
+
"fwd_num_stages": 2,
|
| 57 |
+
"fwd_num_warps": 4,
|
| 58 |
+
"bwd_BLOCK_M1": 32,
|
| 59 |
+
"bwd_BLOCK_N1": 128,
|
| 60 |
+
"bwd_BLOCK_M2": 128,
|
| 61 |
+
"bwd_BLOCK_N2": 32,
|
| 62 |
+
"bwd_num_stages": 3,
|
| 63 |
+
"bwd_num_warps": 8,
|
| 64 |
+
},
|
| 65 |
+
"a100_d256_bwd_64x64": {
|
| 66 |
+
"fwd_BLOCK_M": 64,
|
| 67 |
+
"fwd_BLOCK_N": 64,
|
| 68 |
+
"fwd_num_stages": 3,
|
| 69 |
+
"fwd_num_warps": 4,
|
| 70 |
+
"bwd_BLOCK_M1": 64,
|
| 71 |
+
"bwd_BLOCK_N1": 64,
|
| 72 |
+
"bwd_BLOCK_M2": 64,
|
| 73 |
+
"bwd_BLOCK_N2": 64,
|
| 74 |
+
"bwd_num_stages": 3,
|
| 75 |
+
"bwd_num_warps": 4,
|
| 76 |
+
},
|
| 77 |
+
"a100_d256_bwd_64x128": {
|
| 78 |
+
"fwd_BLOCK_M": 64,
|
| 79 |
+
"fwd_BLOCK_N": 128,
|
| 80 |
+
"fwd_num_stages": 3,
|
| 81 |
+
"fwd_num_warps": 4,
|
| 82 |
+
"bwd_BLOCK_M1": 64,
|
| 83 |
+
"bwd_BLOCK_N1": 128,
|
| 84 |
+
"bwd_BLOCK_M2": 128,
|
| 85 |
+
"bwd_BLOCK_N2": 64,
|
| 86 |
+
"bwd_num_stages": 3,
|
| 87 |
+
"bwd_num_warps": 8,
|
| 88 |
+
},
|
| 89 |
+
"a100_d256_bwd_write_dq_false": {
|
| 90 |
+
"fwd_BLOCK_M": 32,
|
| 91 |
+
"fwd_BLOCK_N": 64,
|
| 92 |
+
"fwd_num_stages": 2,
|
| 93 |
+
"fwd_num_warps": 4,
|
| 94 |
+
"bwd_BLOCK_M1": 32,
|
| 95 |
+
"bwd_BLOCK_N1": 64,
|
| 96 |
+
"bwd_BLOCK_M2": 64,
|
| 97 |
+
"bwd_BLOCK_N2": 32,
|
| 98 |
+
"bwd_num_stages": 3,
|
| 99 |
+
"bwd_num_warps": 4,
|
| 100 |
+
"WRITE_DQ": False,
|
| 101 |
+
},
|
| 102 |
+
"a100_d256_prescale_safe": {
|
| 103 |
+
"fwd_BLOCK_M": 32,
|
| 104 |
+
"fwd_BLOCK_N": 64,
|
| 105 |
+
"fwd_num_stages": 2,
|
| 106 |
+
"fwd_num_warps": 4,
|
| 107 |
+
"bwd_BLOCK_M1": 32,
|
| 108 |
+
"bwd_BLOCK_N1": 64,
|
| 109 |
+
"bwd_BLOCK_M2": 64,
|
| 110 |
+
"bwd_BLOCK_N2": 32,
|
| 111 |
+
"bwd_num_stages": 3,
|
| 112 |
+
"bwd_num_warps": 4,
|
| 113 |
+
"PRESCALE_QK": True,
|
| 114 |
+
"ROWS_GUARANTEED_SAFE": True,
|
| 115 |
+
},
|
| 116 |
+
"a100_d256_contig_safe": {
|
| 117 |
+
"fwd_BLOCK_M": 32,
|
| 118 |
+
"fwd_BLOCK_N": 64,
|
| 119 |
+
"fwd_num_stages": 2,
|
| 120 |
+
"fwd_num_warps": 4,
|
| 121 |
+
"bwd_BLOCK_M1": 32,
|
| 122 |
+
"bwd_BLOCK_N1": 64,
|
| 123 |
+
"bwd_BLOCK_M2": 64,
|
| 124 |
+
"bwd_BLOCK_N2": 32,
|
| 125 |
+
"bwd_num_stages": 3,
|
| 126 |
+
"bwd_num_warps": 4,
|
| 127 |
+
"ROWS_GUARANTEED_SAFE": True,
|
| 128 |
+
"BLOCKS_ARE_CONTIGUOUS": True,
|
| 129 |
+
},
|
| 130 |
+
"a100_d256_contig_prescale": {
|
| 131 |
+
"fwd_BLOCK_M": 32,
|
| 132 |
+
"fwd_BLOCK_N": 64,
|
| 133 |
+
"fwd_num_stages": 2,
|
| 134 |
+
"fwd_num_warps": 4,
|
| 135 |
+
"bwd_BLOCK_M1": 32,
|
| 136 |
+
"bwd_BLOCK_N1": 64,
|
| 137 |
+
"bwd_BLOCK_M2": 64,
|
| 138 |
+
"bwd_BLOCK_N2": 32,
|
| 139 |
+
"bwd_num_stages": 3,
|
| 140 |
+
"bwd_num_warps": 4,
|
| 141 |
+
"PRESCALE_QK": True,
|
| 142 |
+
"ROWS_GUARANTEED_SAFE": True,
|
| 143 |
+
"BLOCKS_ARE_CONTIGUOUS": True,
|
| 144 |
+
},
|
| 145 |
+
"a100_d256_contig_write_dq_false": {
|
| 146 |
+
"fwd_BLOCK_M": 32,
|
| 147 |
+
"fwd_BLOCK_N": 64,
|
| 148 |
+
"fwd_num_stages": 2,
|
| 149 |
+
"fwd_num_warps": 4,
|
| 150 |
+
"bwd_BLOCK_M1": 32,
|
| 151 |
+
"bwd_BLOCK_N1": 64,
|
| 152 |
+
"bwd_BLOCK_M2": 64,
|
| 153 |
+
"bwd_BLOCK_N2": 32,
|
| 154 |
+
"bwd_num_stages": 3,
|
| 155 |
+
"bwd_num_warps": 4,
|
| 156 |
+
"ROWS_GUARANTEED_SAFE": True,
|
| 157 |
+
"BLOCKS_ARE_CONTIGUOUS": True,
|
| 158 |
+
"WRITE_DQ": False,
|
| 159 |
+
},
|
| 160 |
+
}
|
| 161 |
+
|
| 162 |
+
|
| 163 |
+
def bench(fn, warmup: int, iters: int) -> float:
|
| 164 |
+
for _ in range(warmup):
|
| 165 |
+
fn()
|
| 166 |
+
if torch.cuda.is_available():
|
| 167 |
+
torch.cuda.synchronize()
|
| 168 |
+
times = []
|
| 169 |
+
for _ in range(iters):
|
| 170 |
+
if torch.cuda.is_available():
|
| 171 |
+
t0 = torch.cuda.Event(enable_timing=True)
|
| 172 |
+
t1 = torch.cuda.Event(enable_timing=True)
|
| 173 |
+
t0.record()
|
| 174 |
+
fn()
|
| 175 |
+
t1.record()
|
| 176 |
+
torch.cuda.synchronize()
|
| 177 |
+
times.append(t0.elapsed_time(t1))
|
| 178 |
+
else:
|
| 179 |
+
start = time.perf_counter()
|
| 180 |
+
fn()
|
| 181 |
+
times.append((time.perf_counter() - start) * 1000.0)
|
| 182 |
+
times.sort()
|
| 183 |
+
return times[len(times) // 2]
|
| 184 |
+
|
| 185 |
+
|
| 186 |
+
def peak_bytes(fn, device: str) -> int:
|
| 187 |
+
if not device.startswith("cuda"):
|
| 188 |
+
fn()
|
| 189 |
+
return 0
|
| 190 |
+
torch.cuda.reset_peak_memory_stats()
|
| 191 |
+
fn()
|
| 192 |
+
torch.cuda.synchronize()
|
| 193 |
+
return int(torch.cuda.max_memory_allocated())
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
def best_nested_speedup(nested: dict) -> dict | None:
|
| 197 |
+
best = None
|
| 198 |
+
for block_key, by_preset in nested.items():
|
| 199 |
+
for preset, speedup in by_preset.items():
|
| 200 |
+
if not isinstance(speedup, float):
|
| 201 |
+
continue
|
| 202 |
+
if best is None or speedup > best["speedup"]:
|
| 203 |
+
best = {"block_mask": block_key, "preset": preset, "speedup": speedup}
|
| 204 |
+
return best
|
| 205 |
+
|
| 206 |
+
|
| 207 |
+
def make_inputs(args):
|
| 208 |
+
dtype = torch.bfloat16 if args.dtype == "bf16" else torch.float32
|
| 209 |
+
action_len = args.action_blocks * args.action_block_size
|
| 210 |
+
total = args.prefix_len + action_len
|
| 211 |
+
torch.manual_seed(args.seed)
|
| 212 |
+
q = torch.randn(args.batch, args.heads, total, args.head_dim, device=args.device, dtype=dtype)
|
| 213 |
+
kv_heads = args.kv_heads or args.heads
|
| 214 |
+
k = torch.randn(args.batch, kv_heads, total, args.head_dim, device=args.device, dtype=dtype)
|
| 215 |
+
v = torch.randn_like(k)
|
| 216 |
+
prefix_valid = torch.ones(args.batch, args.prefix_len, device=args.device, dtype=torch.bool)
|
| 217 |
+
prefix_att = torch.zeros_like(prefix_valid)
|
| 218 |
+
prefix_att[:, args.prefix_len // 2 :] = True
|
| 219 |
+
if args.no_prefix_mask:
|
| 220 |
+
prefix_valid = None
|
| 221 |
+
prefix_att = None
|
| 222 |
+
return q, k, v, prefix_valid, prefix_att
|
| 223 |
+
|
| 224 |
+
|
| 225 |
+
def make_flex_bundle(args, prefix_valid, prefix_att, block_size: tuple[int, int]):
|
| 226 |
+
try:
|
| 227 |
+
from torch.nn.attention.flex_attention import create_block_mask, flex_attention
|
| 228 |
+
except Exception:
|
| 229 |
+
return None
|
| 230 |
+
|
| 231 |
+
compiled_mask = torch.compile(create_block_mask, dynamic=False)
|
| 232 |
+
batch = args.batch
|
| 233 |
+
prefix_len = args.prefix_len
|
| 234 |
+
action_len = args.action_blocks * args.action_block_size
|
| 235 |
+
total_len = prefix_len + action_len
|
| 236 |
+
chunk = args.action_block_size
|
| 237 |
+
pad = prefix_valid
|
| 238 |
+
cum = torch.cumsum(prefix_att.to(torch.long), dim=1) if prefix_att is not None else None
|
| 239 |
+
|
| 240 |
+
def prefix_rows(b, h, q_idx, kv_idx):
|
| 241 |
+
if cum is None:
|
| 242 |
+
return kv_idx < prefix_len
|
| 243 |
+
kv_p = kv_idx.clamp(max=prefix_len - 1)
|
| 244 |
+
ok = (cum[b, kv_p] <= cum[b, q_idx]) & pad[b, kv_p] & pad[b, q_idx]
|
| 245 |
+
return (kv_idx < prefix_len) & ok
|
| 246 |
+
|
| 247 |
+
def action_rows(b, h, q_idx, kv_idx):
|
| 248 |
+
if pad is None:
|
| 249 |
+
to_prefix = kv_idx < prefix_len
|
| 250 |
+
else:
|
| 251 |
+
kv_p = kv_idx.clamp(max=prefix_len - 1)
|
| 252 |
+
to_prefix = (kv_idx < prefix_len) & pad[b, kv_p]
|
| 253 |
+
same_block = (q_idx // chunk) == ((kv_idx - prefix_len) // chunk)
|
| 254 |
+
return to_prefix | ((kv_idx >= prefix_len) & same_block)
|
| 255 |
+
|
| 256 |
+
block_prefix = compiled_mask(
|
| 257 |
+
prefix_rows,
|
| 258 |
+
B=batch,
|
| 259 |
+
H=None,
|
| 260 |
+
Q_LEN=prefix_len,
|
| 261 |
+
KV_LEN=total_len,
|
| 262 |
+
device=torch.device(args.device),
|
| 263 |
+
BLOCK_SIZE=block_size,
|
| 264 |
+
)
|
| 265 |
+
block_action = compiled_mask(
|
| 266 |
+
action_rows,
|
| 267 |
+
B=batch,
|
| 268 |
+
H=None,
|
| 269 |
+
Q_LEN=action_len,
|
| 270 |
+
KV_LEN=total_len,
|
| 271 |
+
device=torch.device(args.device),
|
| 272 |
+
BLOCK_SIZE=block_size,
|
| 273 |
+
)
|
| 274 |
+
compiled_calls = {}
|
| 275 |
+
scale = args.head_dim**-0.5
|
| 276 |
+
gqa = (args.kv_heads or args.heads) != args.heads
|
| 277 |
+
for name, options in FLEX_TILE_PRESETS.items():
|
| 278 |
+
def prefix_call(q, k, v, block_mask, options=options):
|
| 279 |
+
return flex_attention(
|
| 280 |
+
q,
|
| 281 |
+
k,
|
| 282 |
+
v,
|
| 283 |
+
block_mask=block_mask,
|
| 284 |
+
scale=scale,
|
| 285 |
+
enable_gqa=gqa,
|
| 286 |
+
kernel_options=options,
|
| 287 |
+
)
|
| 288 |
+
|
| 289 |
+
def action_call(q, k, v, block_mask, options=options):
|
| 290 |
+
return flex_attention(
|
| 291 |
+
q,
|
| 292 |
+
k,
|
| 293 |
+
v,
|
| 294 |
+
block_mask=block_mask,
|
| 295 |
+
scale=scale,
|
| 296 |
+
enable_gqa=gqa,
|
| 297 |
+
kernel_options=options,
|
| 298 |
+
)
|
| 299 |
+
|
| 300 |
+
compiled_calls[name] = (
|
| 301 |
+
torch.compile(prefix_call, dynamic=False),
|
| 302 |
+
torch.compile(action_call, dynamic=False),
|
| 303 |
+
)
|
| 304 |
+
return compiled_calls, block_prefix, block_action
|
| 305 |
+
|
| 306 |
+
|
| 307 |
+
def _manual_part(qs, ks, vs, m, scale):
|
| 308 |
+
"""Materialized-logits attention: cuBLAS GEMMs + fused masked softmax.
|
| 309 |
+
|
| 310 |
+
Exact SDPA semantics (fp32 softmax); grouped-query handled as a strided
|
| 311 |
+
batched GEMM over (kv_head, group*Sq) without materializing repeated K/V.
|
| 312 |
+
"""
|
| 313 |
+
B, H, Sq, D = qs.shape
|
| 314 |
+
Hk = ks.shape[1]
|
| 315 |
+
if Hk != H:
|
| 316 |
+
g = H // Hk
|
| 317 |
+
q2 = qs.reshape(B, Hk, g * Sq, D)
|
| 318 |
+
logits = (q2 @ ks.transpose(-1, -2)).reshape(B, H, Sq, -1)
|
| 319 |
+
else:
|
| 320 |
+
logits = qs @ ks.transpose(-1, -2)
|
| 321 |
+
logits = logits * scale + m
|
| 322 |
+
p = logits.float().softmax(dim=-1).to(qs.dtype)
|
| 323 |
+
if Hk != H:
|
| 324 |
+
out = (p.reshape(B, Hk, g * Sq, -1) @ vs).reshape(B, H, Sq, D)
|
| 325 |
+
else:
|
| 326 |
+
out = p @ vs
|
| 327 |
+
return out
|
| 328 |
+
|
| 329 |
+
|
| 330 |
+
_manual_part_compiled = None
|
| 331 |
+
|
| 332 |
+
|
| 333 |
+
def get_manual_part():
|
| 334 |
+
global _manual_part_compiled
|
| 335 |
+
if _manual_part_compiled is None:
|
| 336 |
+
_manual_part_compiled = torch.compile(_manual_part, dynamic=False)
|
| 337 |
+
return _manual_part_compiled
|
| 338 |
+
|
| 339 |
+
|
| 340 |
+
def parse_args() -> argparse.Namespace:
|
| 341 |
+
parser = argparse.ArgumentParser()
|
| 342 |
+
parser.add_argument("--device", default="cuda")
|
| 343 |
+
parser.add_argument("--dtype", choices=["bf16", "fp32"], default="bf16")
|
| 344 |
+
parser.add_argument("--batch", type=int, default=4)
|
| 345 |
+
parser.add_argument("--heads", type=int, default=8)
|
| 346 |
+
parser.add_argument("--kv-heads", type=int, default=None, help="KV heads for GQA (default: same as --heads)")
|
| 347 |
+
parser.add_argument("--head-dim", type=int, default=256)
|
| 348 |
+
parser.add_argument("--prefix-len", type=int, default=700)
|
| 349 |
+
parser.add_argument("--action-blocks", type=int, default=5)
|
| 350 |
+
parser.add_argument("--action-block-size", type=int, default=50)
|
| 351 |
+
parser.add_argument("--warmup", type=int, default=10)
|
| 352 |
+
parser.add_argument("--iters", type=int, default=30)
|
| 353 |
+
parser.add_argument("--seed", type=int, default=0)
|
| 354 |
+
parser.add_argument("--mode", choices=["fwd", "fwdbwd", "all"], default="all")
|
| 355 |
+
parser.add_argument(
|
| 356 |
+
"--backend",
|
| 357 |
+
default="all",
|
| 358 |
+
help="comma-separated subset of {package, torch-flex, manual} or 'all'",
|
| 359 |
+
)
|
| 360 |
+
parser.add_argument("--flex-preset", choices=sorted(FLEX_TILE_PRESETS), default="a100_d256_bwd_32x64")
|
| 361 |
+
parser.add_argument("--sweep-flex-presets", action="store_true")
|
| 362 |
+
parser.add_argument("--block-mask-q", type=int, default=128)
|
| 363 |
+
parser.add_argument("--block-mask-kv", type=int, default=128)
|
| 364 |
+
parser.add_argument("--sweep-block-mask-sizes", action="store_true")
|
| 365 |
+
parser.add_argument("--output")
|
| 366 |
+
parser.add_argument("--require-gates", action="store_true")
|
| 367 |
+
parser.add_argument("--no-prefix-mask", action="store_true")
|
| 368 |
+
return parser.parse_args()
|
| 369 |
+
|
| 370 |
+
|
| 371 |
+
def main() -> None:
|
| 372 |
+
args = parse_args()
|
| 373 |
+
if args.device.startswith("cuda") and not torch.cuda.is_available():
|
| 374 |
+
raise SystemExit("CUDA requested but not available")
|
| 375 |
+
backends = (
|
| 376 |
+
{"package", "torch-flex", "manual"}
|
| 377 |
+
if args.backend == "all"
|
| 378 |
+
else {b.strip() for b in args.backend.split(",") if b.strip()}
|
| 379 |
+
)
|
| 380 |
+
|
| 381 |
+
q, k, v, prefix_valid, prefix_att = make_inputs(args)
|
| 382 |
+
action_len = args.action_blocks * args.action_block_size
|
| 383 |
+
scale = args.head_dim**-0.5
|
| 384 |
+
pm, am = flex_ops.build_block_sparse_bool_masks(
|
| 385 |
+
prefix_valid,
|
| 386 |
+
prefix_att,
|
| 387 |
+
batch=args.batch,
|
| 388 |
+
prefix_len=args.prefix_len,
|
| 389 |
+
action_len=action_len,
|
| 390 |
+
action_block_size=args.action_block_size,
|
| 391 |
+
device=q.device,
|
| 392 |
+
)
|
| 393 |
+
full = torch.cat([pm, am], dim=1)
|
| 394 |
+
add_mask = torch.where(
|
| 395 |
+
full[:, None],
|
| 396 |
+
torch.zeros((), device=q.device, dtype=q.dtype),
|
| 397 |
+
torch.full((), flex_ops.MASK_VALUE_F32, device=q.device, dtype=q.dtype),
|
| 398 |
+
)
|
| 399 |
+
block_sizes = [(args.block_mask_q, args.block_mask_kv)]
|
| 400 |
+
if args.sweep_block_mask_sizes:
|
| 401 |
+
block_sizes = [(q, kv) for q in (16, 32, 64, 128) for kv in (32, 64, 128)]
|
| 402 |
+
flex_bundles = {
|
| 403 |
+
f"{q}x{kv}": make_flex_bundle(args, prefix_valid, prefix_att, (q, kv))
|
| 404 |
+
for q, kv in block_sizes
|
| 405 |
+
} if "torch-flex" in backends else {}
|
| 406 |
+
flex_presets = sorted(FLEX_TILE_PRESETS) if args.sweep_flex_presets else [args.flex_preset]
|
| 407 |
+
|
| 408 |
+
gqa = (args.kv_heads or args.heads) != args.heads
|
| 409 |
+
|
| 410 |
+
def sdpa_fwd():
|
| 411 |
+
out_p = F.scaled_dot_product_attention(
|
| 412 |
+
q[:, :, : args.prefix_len],
|
| 413 |
+
k,
|
| 414 |
+
v,
|
| 415 |
+
attn_mask=add_mask[:, :, : args.prefix_len],
|
| 416 |
+
scale=scale,
|
| 417 |
+
enable_gqa=gqa,
|
| 418 |
+
)
|
| 419 |
+
kd = torch.cat([k[:, :, : args.prefix_len].detach(), k[:, :, args.prefix_len :]], dim=2)
|
| 420 |
+
vd = torch.cat([v[:, :, : args.prefix_len].detach(), v[:, :, args.prefix_len :]], dim=2)
|
| 421 |
+
out_a = F.scaled_dot_product_attention(
|
| 422 |
+
q[:, :, args.prefix_len :],
|
| 423 |
+
kd,
|
| 424 |
+
vd,
|
| 425 |
+
attn_mask=add_mask[:, :, args.prefix_len :],
|
| 426 |
+
scale=scale,
|
| 427 |
+
enable_gqa=gqa,
|
| 428 |
+
)
|
| 429 |
+
return torch.cat([out_p, out_a], dim=2)
|
| 430 |
+
|
| 431 |
+
manual_part = get_manual_part() if "manual" in backends else None
|
| 432 |
+
|
| 433 |
+
def manual_eager_fwd():
|
| 434 |
+
out_p = _manual_part(q[:, :, : args.prefix_len], k, v, add_mask[:, :, : args.prefix_len], scale)
|
| 435 |
+
kd = torch.cat([k[:, :, : args.prefix_len].detach(), k[:, :, args.prefix_len :]], dim=2)
|
| 436 |
+
vd = torch.cat([v[:, :, : args.prefix_len].detach(), v[:, :, args.prefix_len :]], dim=2)
|
| 437 |
+
out_a = _manual_part(
|
| 438 |
+
q[:, :, args.prefix_len :], kd, vd, add_mask[:, :, args.prefix_len :], scale
|
| 439 |
+
)
|
| 440 |
+
return torch.cat([out_p, out_a], dim=2)
|
| 441 |
+
|
| 442 |
+
def manual_eager_fwdbwd():
|
| 443 |
+
qq = q.detach().clone().requires_grad_(True)
|
| 444 |
+
kk = k.detach().clone().requires_grad_(True)
|
| 445 |
+
vv = v.detach().clone().requires_grad_(True)
|
| 446 |
+
out_p = _manual_part(
|
| 447 |
+
qq[:, :, : args.prefix_len], kk, vv, add_mask[:, :, : args.prefix_len], scale
|
| 448 |
+
)
|
| 449 |
+
kd = torch.cat([kk[:, :, : args.prefix_len].detach(), kk[:, :, args.prefix_len :]], dim=2)
|
| 450 |
+
vd = torch.cat([vv[:, :, : args.prefix_len].detach(), vv[:, :, args.prefix_len :]], dim=2)
|
| 451 |
+
out_a = _manual_part(
|
| 452 |
+
qq[:, :, args.prefix_len :], kd, vd, add_mask[:, :, args.prefix_len :], scale
|
| 453 |
+
)
|
| 454 |
+
torch.cat([out_p, out_a], dim=2).float().square().mean().backward()
|
| 455 |
+
|
| 456 |
+
def _repeat_kv(t):
|
| 457 |
+
# the model's current path: materialize K/V to the q-head count
|
| 458 |
+
return t.repeat_interleave(args.heads // t.shape[1], dim=1)
|
| 459 |
+
|
| 460 |
+
def sdpa_repeat_fwd():
|
| 461 |
+
kr, vr = _repeat_kv(k), _repeat_kv(v)
|
| 462 |
+
out_p = F.scaled_dot_product_attention(
|
| 463 |
+
q[:, :, : args.prefix_len], kr, vr, attn_mask=add_mask[:, :, : args.prefix_len], scale=scale
|
| 464 |
+
)
|
| 465 |
+
kd = torch.cat([kr[:, :, : args.prefix_len].detach(), kr[:, :, args.prefix_len :]], dim=2)
|
| 466 |
+
vd = torch.cat([vr[:, :, : args.prefix_len].detach(), vr[:, :, args.prefix_len :]], dim=2)
|
| 467 |
+
out_a = F.scaled_dot_product_attention(
|
| 468 |
+
q[:, :, args.prefix_len :], kd, vd, attn_mask=add_mask[:, :, args.prefix_len :], scale=scale
|
| 469 |
+
)
|
| 470 |
+
return torch.cat([out_p, out_a], dim=2)
|
| 471 |
+
|
| 472 |
+
def manual_fwd():
|
| 473 |
+
out_p = manual_part(q[:, :, : args.prefix_len], k, v, add_mask[:, :, : args.prefix_len], scale)
|
| 474 |
+
kd = torch.cat([k[:, :, : args.prefix_len].detach(), k[:, :, args.prefix_len :]], dim=2)
|
| 475 |
+
vd = torch.cat([v[:, :, : args.prefix_len].detach(), v[:, :, args.prefix_len :]], dim=2)
|
| 476 |
+
out_a = manual_part(
|
| 477 |
+
q[:, :, args.prefix_len :], kd, vd, add_mask[:, :, args.prefix_len :], scale
|
| 478 |
+
)
|
| 479 |
+
return torch.cat([out_p, out_a], dim=2)
|
| 480 |
+
|
| 481 |
+
def package_fwd():
|
| 482 |
+
return flex_ops.flex_attention(
|
| 483 |
+
q,
|
| 484 |
+
k,
|
| 485 |
+
v,
|
| 486 |
+
prefix_len=args.prefix_len,
|
| 487 |
+
action_block_size=args.action_block_size,
|
| 488 |
+
prefix_valid=prefix_valid,
|
| 489 |
+
prefix_att=prefix_att,
|
| 490 |
+
scale=scale,
|
| 491 |
+
)
|
| 492 |
+
|
| 493 |
+
def torch_flex_fwd(preset: str, block_key: str):
|
| 494 |
+
if not flex_bundles:
|
| 495 |
+
raise RuntimeError("PyTorch FlexAttention is unavailable")
|
| 496 |
+
compiled_calls, block_prefix, block_action = flex_bundles[block_key]
|
| 497 |
+
prefix_call, action_call = compiled_calls[preset]
|
| 498 |
+
out_p = prefix_call(
|
| 499 |
+
q[:, :, : args.prefix_len],
|
| 500 |
+
k,
|
| 501 |
+
v,
|
| 502 |
+
block_prefix,
|
| 503 |
+
)
|
| 504 |
+
kd = torch.cat([k[:, :, : args.prefix_len].detach(), k[:, :, args.prefix_len :]], dim=2)
|
| 505 |
+
vd = torch.cat([v[:, :, : args.prefix_len].detach(), v[:, :, args.prefix_len :]], dim=2)
|
| 506 |
+
out_a = action_call(
|
| 507 |
+
q[:, :, args.prefix_len :],
|
| 508 |
+
kd,
|
| 509 |
+
vd,
|
| 510 |
+
block_action,
|
| 511 |
+
)
|
| 512 |
+
return torch.cat([out_p, out_a], dim=2)
|
| 513 |
+
|
| 514 |
+
report = {
|
| 515 |
+
"gpu": torch.cuda.get_device_name() if args.device.startswith("cuda") else "cpu",
|
| 516 |
+
"torch": torch.__version__,
|
| 517 |
+
"shape": {
|
| 518 |
+
"B": args.batch,
|
| 519 |
+
"heads": args.heads,
|
| 520 |
+
"kv_heads": args.kv_heads or args.heads,
|
| 521 |
+
"head_dim": args.head_dim,
|
| 522 |
+
"prefix_len": args.prefix_len,
|
| 523 |
+
"action_len": action_len,
|
| 524 |
+
"action_block_size": args.action_block_size,
|
| 525 |
+
},
|
| 526 |
+
"backend": args.backend,
|
| 527 |
+
"flex_presets": flex_presets,
|
| 528 |
+
"block_mask_sizes": list(flex_bundles) if flex_bundles else [],
|
| 529 |
+
"timestamp": time.strftime("%Y-%m-%d %H:%M:%S"),
|
| 530 |
+
}
|
| 531 |
+
report["package_has_native_ops"] = False
|
| 532 |
+
report["package_native_supported"] = False
|
| 533 |
+
|
| 534 |
+
with torch.no_grad():
|
| 535 |
+
sdpa_out = sdpa_fwd().float()
|
| 536 |
+
if "package" in backends:
|
| 537 |
+
report["package_fwd_max_abs_diff"] = float((sdpa_out - package_fwd().float()).abs().max())
|
| 538 |
+
if "manual" in backends:
|
| 539 |
+
report["manual_fwd_max_abs_diff"] = float((sdpa_out - manual_fwd().float()).abs().max())
|
| 540 |
+
if flex_bundles:
|
| 541 |
+
report["torch_flex_fwd_max_abs_diff"] = {}
|
| 542 |
+
for block_key in flex_bundles:
|
| 543 |
+
report["torch_flex_fwd_max_abs_diff"][block_key] = {}
|
| 544 |
+
for block_key in flex_bundles:
|
| 545 |
+
for preset in flex_presets:
|
| 546 |
+
try:
|
| 547 |
+
report["torch_flex_fwd_max_abs_diff"][block_key][preset] = float(
|
| 548 |
+
(sdpa_out - torch_flex_fwd(preset, block_key).float()).abs().max()
|
| 549 |
+
)
|
| 550 |
+
except Exception as exc:
|
| 551 |
+
report["torch_flex_fwd_max_abs_diff"][block_key][preset] = f"ERROR: {type(exc).__name__}: {exc}"
|
| 552 |
+
if args.mode in {"fwd", "all"}:
|
| 553 |
+
report["sdpa_fwd_ms"] = bench(sdpa_fwd, args.warmup, args.iters)
|
| 554 |
+
report["sdpa_peak_bytes"] = peak_bytes(sdpa_fwd, args.device)
|
| 555 |
+
if "package" in backends:
|
| 556 |
+
report["package_fwd_ms"] = bench(package_fwd, args.warmup, args.iters)
|
| 557 |
+
report["package_fwd_speedup"] = report["sdpa_fwd_ms"] / report["package_fwd_ms"]
|
| 558 |
+
report["package_peak_bytes"] = peak_bytes(package_fwd, args.device)
|
| 559 |
+
if "manual" in backends:
|
| 560 |
+
report["manual_fwd_ms"] = bench(manual_fwd, args.warmup, args.iters)
|
| 561 |
+
report["manual_fwd_speedup"] = report["sdpa_fwd_ms"] / report["manual_fwd_ms"]
|
| 562 |
+
report["manual_peak_bytes"] = peak_bytes(manual_fwd, args.device)
|
| 563 |
+
if gqa:
|
| 564 |
+
report["sdpa_repeat_fwd_ms"] = bench(sdpa_repeat_fwd, args.warmup, args.iters)
|
| 565 |
+
if "manual" in backends:
|
| 566 |
+
report["manual_eager_fwd_ms"] = bench(manual_eager_fwd, args.warmup, args.iters)
|
| 567 |
+
if flex_bundles:
|
| 568 |
+
report["torch_flex_fwd_ms"] = {}
|
| 569 |
+
report["torch_flex_fwd_speedup"] = {}
|
| 570 |
+
for block_key in flex_bundles:
|
| 571 |
+
report["torch_flex_fwd_ms"][block_key] = {}
|
| 572 |
+
report["torch_flex_fwd_speedup"][block_key] = {}
|
| 573 |
+
for block_key in flex_bundles:
|
| 574 |
+
for preset in flex_presets:
|
| 575 |
+
try:
|
| 576 |
+
ms = bench(lambda preset=preset, block_key=block_key: torch_flex_fwd(preset, block_key), args.warmup, args.iters)
|
| 577 |
+
except Exception as exc:
|
| 578 |
+
report["torch_flex_fwd_ms"][block_key][preset] = f"ERROR: {type(exc).__name__}: {exc}"
|
| 579 |
+
else:
|
| 580 |
+
report["torch_flex_fwd_ms"][block_key][preset] = ms
|
| 581 |
+
report["torch_flex_fwd_speedup"][block_key][preset] = report["sdpa_fwd_ms"] / ms
|
| 582 |
+
report["best_torch_flex_fwd"] = best_nested_speedup(report["torch_flex_fwd_speedup"])
|
| 583 |
+
|
| 584 |
+
if args.mode in {"fwdbwd", "all"}:
|
| 585 |
+
def sdpa_fwdbwd():
|
| 586 |
+
qq = q.detach().clone().requires_grad_(True)
|
| 587 |
+
kk = k.detach().clone().requires_grad_(True)
|
| 588 |
+
vv = v.detach().clone().requires_grad_(True)
|
| 589 |
+
out_p = F.scaled_dot_product_attention(
|
| 590 |
+
qq[:, :, : args.prefix_len],
|
| 591 |
+
kk,
|
| 592 |
+
vv,
|
| 593 |
+
attn_mask=add_mask[:, :, : args.prefix_len],
|
| 594 |
+
scale=scale,
|
| 595 |
+
enable_gqa=gqa,
|
| 596 |
+
)
|
| 597 |
+
kd = torch.cat([kk[:, :, : args.prefix_len].detach(), kk[:, :, args.prefix_len :]], dim=2)
|
| 598 |
+
vd = torch.cat([vv[:, :, : args.prefix_len].detach(), vv[:, :, args.prefix_len :]], dim=2)
|
| 599 |
+
out_a = F.scaled_dot_product_attention(
|
| 600 |
+
qq[:, :, args.prefix_len :],
|
| 601 |
+
kd,
|
| 602 |
+
vd,
|
| 603 |
+
attn_mask=add_mask[:, :, args.prefix_len :],
|
| 604 |
+
scale=scale,
|
| 605 |
+
enable_gqa=gqa,
|
| 606 |
+
)
|
| 607 |
+
torch.cat([out_p, out_a], dim=2).float().square().mean().backward()
|
| 608 |
+
|
| 609 |
+
def manual_fwdbwd():
|
| 610 |
+
qq = q.detach().clone().requires_grad_(True)
|
| 611 |
+
kk = k.detach().clone().requires_grad_(True)
|
| 612 |
+
vv = v.detach().clone().requires_grad_(True)
|
| 613 |
+
out_p = manual_part(
|
| 614 |
+
qq[:, :, : args.prefix_len], kk, vv, add_mask[:, :, : args.prefix_len], scale
|
| 615 |
+
)
|
| 616 |
+
kd = torch.cat([kk[:, :, : args.prefix_len].detach(), kk[:, :, args.prefix_len :]], dim=2)
|
| 617 |
+
vd = torch.cat([vv[:, :, : args.prefix_len].detach(), vv[:, :, args.prefix_len :]], dim=2)
|
| 618 |
+
out_a = manual_part(
|
| 619 |
+
qq[:, :, args.prefix_len :], kd, vd, add_mask[:, :, args.prefix_len :], scale
|
| 620 |
+
)
|
| 621 |
+
torch.cat([out_p, out_a], dim=2).float().square().mean().backward()
|
| 622 |
+
|
| 623 |
+
def sdpa_repeat_fwdbwd():
|
| 624 |
+
qq = q.detach().clone().requires_grad_(True)
|
| 625 |
+
kk = k.detach().clone().requires_grad_(True)
|
| 626 |
+
vv = v.detach().clone().requires_grad_(True)
|
| 627 |
+
kr, vr = _repeat_kv(kk), _repeat_kv(vv)
|
| 628 |
+
out_p = F.scaled_dot_product_attention(
|
| 629 |
+
qq[:, :, : args.prefix_len], kr, vr, attn_mask=add_mask[:, :, : args.prefix_len], scale=scale
|
| 630 |
+
)
|
| 631 |
+
kd = torch.cat([kr[:, :, : args.prefix_len].detach(), kr[:, :, args.prefix_len :]], dim=2)
|
| 632 |
+
vd = torch.cat([vr[:, :, : args.prefix_len].detach(), vr[:, :, args.prefix_len :]], dim=2)
|
| 633 |
+
out_a = F.scaled_dot_product_attention(
|
| 634 |
+
qq[:, :, args.prefix_len :], kd, vd, attn_mask=add_mask[:, :, args.prefix_len :], scale=scale
|
| 635 |
+
)
|
| 636 |
+
torch.cat([out_p, out_a], dim=2).float().square().mean().backward()
|
| 637 |
+
|
| 638 |
+
def package_fwdbwd():
|
| 639 |
+
qq = q.detach().clone().requires_grad_(True)
|
| 640 |
+
kk = k.detach().clone().requires_grad_(True)
|
| 641 |
+
vv = v.detach().clone().requires_grad_(True)
|
| 642 |
+
flex_ops.flex_attention(
|
| 643 |
+
qq,
|
| 644 |
+
kk,
|
| 645 |
+
vv,
|
| 646 |
+
prefix_len=args.prefix_len,
|
| 647 |
+
action_block_size=args.action_block_size,
|
| 648 |
+
prefix_valid=prefix_valid,
|
| 649 |
+
prefix_att=prefix_att,
|
| 650 |
+
scale=scale,
|
| 651 |
+
).float().square().mean().backward()
|
| 652 |
+
|
| 653 |
+
def torch_flex_fwdbwd(preset: str, block_key: str):
|
| 654 |
+
if not flex_bundles:
|
| 655 |
+
raise RuntimeError("PyTorch FlexAttention is unavailable")
|
| 656 |
+
compiled_calls, block_prefix, block_action = flex_bundles[block_key]
|
| 657 |
+
prefix_call, action_call = compiled_calls[preset]
|
| 658 |
+
qq = q.detach().clone().requires_grad_(True)
|
| 659 |
+
kk = k.detach().clone().requires_grad_(True)
|
| 660 |
+
vv = v.detach().clone().requires_grad_(True)
|
| 661 |
+
out_p = prefix_call(
|
| 662 |
+
qq[:, :, : args.prefix_len],
|
| 663 |
+
kk,
|
| 664 |
+
vv,
|
| 665 |
+
block_prefix,
|
| 666 |
+
)
|
| 667 |
+
kd = torch.cat([kk[:, :, : args.prefix_len].detach(), kk[:, :, args.prefix_len :]], dim=2)
|
| 668 |
+
vd = torch.cat([vv[:, :, : args.prefix_len].detach(), vv[:, :, args.prefix_len :]], dim=2)
|
| 669 |
+
out_a = action_call(
|
| 670 |
+
qq[:, :, args.prefix_len :],
|
| 671 |
+
kd,
|
| 672 |
+
vd,
|
| 673 |
+
block_action,
|
| 674 |
+
)
|
| 675 |
+
torch.cat([out_p, out_a], dim=2).float().square().mean().backward()
|
| 676 |
+
|
| 677 |
+
report["sdpa_fwdbwd_ms"] = bench(sdpa_fwdbwd, args.warmup, args.iters)
|
| 678 |
+
if "package" in backends:
|
| 679 |
+
report["package_fwdbwd_ms"] = bench(package_fwdbwd, args.warmup, args.iters)
|
| 680 |
+
report["package_fwdbwd_speedup"] = report["sdpa_fwdbwd_ms"] / report["package_fwdbwd_ms"]
|
| 681 |
+
if "manual" in backends:
|
| 682 |
+
report["manual_fwdbwd_ms"] = bench(manual_fwdbwd, args.warmup, args.iters)
|
| 683 |
+
report["manual_fwdbwd_speedup"] = report["sdpa_fwdbwd_ms"] / report["manual_fwdbwd_ms"]
|
| 684 |
+
if gqa:
|
| 685 |
+
report["sdpa_repeat_fwdbwd_ms"] = bench(sdpa_repeat_fwdbwd, args.warmup, args.iters)
|
| 686 |
+
if "manual" in backends:
|
| 687 |
+
report["manual_eager_fwdbwd_ms"] = bench(manual_eager_fwdbwd, args.warmup, args.iters)
|
| 688 |
+
if flex_bundles:
|
| 689 |
+
report["torch_flex_fwdbwd_ms"] = {}
|
| 690 |
+
report["torch_flex_fwdbwd_speedup"] = {}
|
| 691 |
+
for block_key in flex_bundles:
|
| 692 |
+
report["torch_flex_fwdbwd_ms"][block_key] = {}
|
| 693 |
+
report["torch_flex_fwdbwd_speedup"][block_key] = {}
|
| 694 |
+
for block_key in flex_bundles:
|
| 695 |
+
for preset in flex_presets:
|
| 696 |
+
try:
|
| 697 |
+
ms = bench(lambda preset=preset, block_key=block_key: torch_flex_fwdbwd(preset, block_key), args.warmup, args.iters)
|
| 698 |
+
except Exception as exc:
|
| 699 |
+
report["torch_flex_fwdbwd_ms"][block_key][preset] = f"ERROR: {type(exc).__name__}: {exc}"
|
| 700 |
+
else:
|
| 701 |
+
report["torch_flex_fwdbwd_ms"][block_key][preset] = ms
|
| 702 |
+
report["torch_flex_fwdbwd_speedup"][block_key][preset] = report["sdpa_fwdbwd_ms"] / ms
|
| 703 |
+
report["best_torch_flex_fwdbwd"] = best_nested_speedup(report["torch_flex_fwdbwd_speedup"])
|
| 704 |
+
|
| 705 |
+
if "package" not in backends:
|
| 706 |
+
_finish(report, args)
|
| 707 |
+
return
|
| 708 |
+
q1 = q.detach().clone().requires_grad_(True)
|
| 709 |
+
k1 = k.detach().clone().requires_grad_(True)
|
| 710 |
+
v1 = v.detach().clone().requires_grad_(True)
|
| 711 |
+
q2 = q.detach().clone().requires_grad_(True)
|
| 712 |
+
k2 = k.detach().clone().requires_grad_(True)
|
| 713 |
+
v2 = v.detach().clone().requires_grad_(True)
|
| 714 |
+
out1_p = F.scaled_dot_product_attention(
|
| 715 |
+
q1[:, :, : args.prefix_len], k1, v1, attn_mask=add_mask[:, :, : args.prefix_len], scale=scale
|
| 716 |
+
)
|
| 717 |
+
k1d = torch.cat([k1[:, :, : args.prefix_len].detach(), k1[:, :, args.prefix_len :]], dim=2)
|
| 718 |
+
v1d = torch.cat([v1[:, :, : args.prefix_len].detach(), v1[:, :, args.prefix_len :]], dim=2)
|
| 719 |
+
out1_a = F.scaled_dot_product_attention(
|
| 720 |
+
q1[:, :, args.prefix_len :], k1d, v1d, attn_mask=add_mask[:, :, args.prefix_len :], scale=scale
|
| 721 |
+
)
|
| 722 |
+
torch.cat([out1_p, out1_a], dim=2).float().square().mean().backward()
|
| 723 |
+
flex_ops.flex_attention(
|
| 724 |
+
q2,
|
| 725 |
+
k2,
|
| 726 |
+
v2,
|
| 727 |
+
prefix_len=args.prefix_len,
|
| 728 |
+
action_block_size=args.action_block_size,
|
| 729 |
+
prefix_valid=prefix_valid,
|
| 730 |
+
prefix_att=prefix_att,
|
| 731 |
+
scale=scale,
|
| 732 |
+
).float().square().mean().backward()
|
| 733 |
+
denom = torch.linalg.vector_norm(torch.cat([q1.grad.flatten(), k1.grad.flatten(), v1.grad.flatten()])).clamp_min(1e-12)
|
| 734 |
+
numer = torch.linalg.vector_norm(
|
| 735 |
+
torch.cat([(q1.grad - q2.grad).flatten(), (k1.grad - k2.grad).flatten(), (v1.grad - v2.grad).flatten()])
|
| 736 |
+
)
|
| 737 |
+
report["package_grad_norm_rel_diff"] = float(numer / denom)
|
| 738 |
+
|
| 739 |
+
_finish(report, args)
|
| 740 |
+
|
| 741 |
+
|
| 742 |
+
def _finish(report, args):
|
| 743 |
+
gates = {
|
| 744 |
+
"package_fwd_max_abs_diff": report.get("package_fwd_max_abs_diff", 0.0) <= 2e-3,
|
| 745 |
+
"package_grad_norm_rel_diff": report.get("package_grad_norm_rel_diff", 0.0) <= 1e-2,
|
| 746 |
+
}
|
| 747 |
+
if "torch_flex_fwd_max_abs_diff" in report:
|
| 748 |
+
vals = [v for by_block in report["torch_flex_fwd_max_abs_diff"].values() for v in by_block.values()]
|
| 749 |
+
gates["torch_flex_fwd_max_abs_diff"] = any(isinstance(v, float) and v <= 2e-3 for v in vals)
|
| 750 |
+
if "manual_fwd_max_abs_diff" in report:
|
| 751 |
+
gates["manual_fwd_max_abs_diff"] = report["manual_fwd_max_abs_diff"] <= 2e-3
|
| 752 |
+
if "manual_fwd_speedup" in report:
|
| 753 |
+
gates["manual_fwd_speedup_ge_1p0526"] = report["manual_fwd_speedup"] >= (1.0 / 0.95)
|
| 754 |
+
if "manual_fwdbwd_speedup" in report:
|
| 755 |
+
gates["manual_fwdbwd_speedup_ge_1p0526"] = report["manual_fwdbwd_speedup"] >= (1.0 / 0.95)
|
| 756 |
+
if "package_fwd_speedup" in report:
|
| 757 |
+
gates["package_fwd_speedup_ge_1p0526"] = report["package_fwd_speedup"] >= (1.0 / 0.95)
|
| 758 |
+
if report.get("sdpa_peak_bytes", 0) > 0:
|
| 759 |
+
gates["package_peak_memory_le_sdpa_plus_3pct"] = report["package_peak_bytes"] <= int(report["sdpa_peak_bytes"] * 1.03)
|
| 760 |
+
if "package_fwdbwd_speedup" in report:
|
| 761 |
+
gates["package_fwdbwd_speedup_ge_1p0526"] = report["package_fwdbwd_speedup"] >= (1.0 / 0.95)
|
| 762 |
+
if "torch_flex_fwd_speedup" in report:
|
| 763 |
+
vals = [v for by_block in report["torch_flex_fwd_speedup"].values() for v in by_block.values() if isinstance(v, float)]
|
| 764 |
+
gates["torch_flex_fwd_speedup_ge_1p0526"] = bool(vals) and max(vals) >= (1.0 / 0.95)
|
| 765 |
+
if "torch_flex_fwdbwd_speedup" in report:
|
| 766 |
+
vals = [v for by_block in report["torch_flex_fwdbwd_speedup"].values() for v in by_block.values() if isinstance(v, float)]
|
| 767 |
+
gates["torch_flex_fwdbwd_speedup_ge_1p0526"] = bool(vals) and max(vals) >= (1.0 / 0.95)
|
| 768 |
+
report["gates"] = gates
|
| 769 |
+
|
| 770 |
+
text = json.dumps(report, indent=2)
|
| 771 |
+
if args.output:
|
| 772 |
+
Path(args.output).write_text(text + "\n", encoding="utf-8")
|
| 773 |
+
print(text)
|
| 774 |
+
if args.require_gates and not all(gates.values()):
|
| 775 |
+
raise SystemExit("one or more Flex attention acceptance gates failed")
|
| 776 |
+
|
| 777 |
+
|
| 778 |
+
if __name__ == "__main__":
|
| 779 |
+
main()
|
build/torch211-cxx11-cu128-x86_64-linux/__init__.py
ADDED
|
@@ -0,0 +1,487 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""FlashRT Flex-style block-sparse attention training API.
|
| 2 |
+
|
| 3 |
+
The public function implements the PI052 prefix/action mask pattern:
|
| 4 |
+
|
| 5 |
+
* prefix query rows use the original K/V tensors, so prefix losses keep normal
|
| 6 |
+
gradients into prefix K/V;
|
| 7 |
+
* action query rows read detached prefix K/V plus normal action K/V by default,
|
| 8 |
+
matching the current training semantics.
|
| 9 |
+
|
| 10 |
+
Unsupported shapes route to the SDPA reference path. Native CUDA kernels are
|
| 11 |
+
not exposed until a shape-specialized implementation beats SDPA on the target
|
| 12 |
+
A100/5090 validation matrix.
|
| 13 |
+
"""
|
| 14 |
+
|
| 15 |
+
from __future__ import annotations
|
| 16 |
+
|
| 17 |
+
from typing import Optional
|
| 18 |
+
|
| 19 |
+
import torch
|
| 20 |
+
import torch.nn.functional as F
|
| 21 |
+
|
| 22 |
+
try:
|
| 23 |
+
from ._ops import ops
|
| 24 |
+
|
| 25 |
+
_HAS_OPS = hasattr(ops, "_flashrt_training_package_marker")
|
| 26 |
+
except Exception: # source-tree tests before kernel-builder creates _ops.py
|
| 27 |
+
ops = None
|
| 28 |
+
_HAS_OPS = False
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
MASK_VALUE_F32 = -2.3819763e38
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def _use_ops(namespace_ops) -> None:
|
| 35 |
+
"""Install a manually built extension (dev/testing path)."""
|
| 36 |
+
global ops, _HAS_OPS
|
| 37 |
+
ops = namespace_ops
|
| 38 |
+
_HAS_OPS = hasattr(ops, "_flashrt_training_package_marker")
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def _check_qkv(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor) -> None:
|
| 42 |
+
if q.dim() != 4 or k.dim() != 4 or v.dim() != 4:
|
| 43 |
+
raise ValueError("q, k, and v must be shaped (B, H, S, D)")
|
| 44 |
+
if q.shape[0] != k.shape[0] or q.shape[0] != v.shape[0]:
|
| 45 |
+
raise ValueError("q, k, and v batch dimensions must match")
|
| 46 |
+
if k.shape != v.shape:
|
| 47 |
+
raise ValueError("k and v shapes must match")
|
| 48 |
+
if q.shape[2] != k.shape[2] or q.shape[3] != k.shape[3]:
|
| 49 |
+
raise ValueError("q, k, and v sequence/head_dim dimensions must match")
|
| 50 |
+
if q.device != k.device or q.device != v.device:
|
| 51 |
+
raise ValueError("q, k, and v must be on the same device")
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def _as_valid(mask: Optional[torch.Tensor], batch: int, length: int, device: torch.device) -> torch.Tensor:
|
| 55 |
+
if mask is None:
|
| 56 |
+
return torch.ones((batch, length), dtype=torch.bool, device=device)
|
| 57 |
+
if mask.shape != (batch, length):
|
| 58 |
+
raise ValueError(f"mask must be shaped {(batch, length)}, got {tuple(mask.shape)}")
|
| 59 |
+
return mask.to(device=device, dtype=torch.bool)
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def build_block_sparse_bool_masks(
|
| 63 |
+
prefix_valid: Optional[torch.Tensor],
|
| 64 |
+
prefix_att: Optional[torch.Tensor],
|
| 65 |
+
*,
|
| 66 |
+
batch: int,
|
| 67 |
+
prefix_len: int,
|
| 68 |
+
action_len: int,
|
| 69 |
+
action_block_size: int,
|
| 70 |
+
non_fast_prefix_len: Optional[int] = None,
|
| 71 |
+
action_valid: Optional[torch.Tensor] = None,
|
| 72 |
+
device: Optional[torch.device] = None,
|
| 73 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 74 |
+
"""Build boolean masks for the split FlexAttention SDPA calls.
|
| 75 |
+
|
| 76 |
+
Returns ``(prefix_rows, action_rows)`` with shapes ``(B, P, S)`` and
|
| 77 |
+
``(B, A, S)``. Boolean True means the key/value position is visible.
|
| 78 |
+
|
| 79 |
+
``prefix_att`` follows Lerobot's cumulative-block convention: prefix key
|
| 80 |
+
``j`` is visible to prefix query ``i`` when ``cumsum(prefix_att)[j] <=
|
| 81 |
+
cumsum(prefix_att)[i]`` and both rows are valid. When omitted, prefix rows
|
| 82 |
+
attend to all valid prefix tokens.
|
| 83 |
+
"""
|
| 84 |
+
if action_block_size <= 0:
|
| 85 |
+
raise ValueError("action_block_size must be positive")
|
| 86 |
+
if prefix_len < 0 or action_len < 0:
|
| 87 |
+
raise ValueError("prefix_len and action_len must be non-negative")
|
| 88 |
+
total_len = prefix_len + action_len
|
| 89 |
+
dev = device
|
| 90 |
+
if dev is None:
|
| 91 |
+
for t in (prefix_valid, prefix_att, action_valid):
|
| 92 |
+
if t is not None:
|
| 93 |
+
dev = t.device
|
| 94 |
+
break
|
| 95 |
+
if dev is None:
|
| 96 |
+
dev = torch.device("cpu")
|
| 97 |
+
|
| 98 |
+
p_valid = _as_valid(prefix_valid, batch, prefix_len, dev)
|
| 99 |
+
a_valid = _as_valid(action_valid, batch, action_len, dev)
|
| 100 |
+
|
| 101 |
+
if prefix_att is None:
|
| 102 |
+
prefix_rows = p_valid[:, :, None] & p_valid[:, None, :]
|
| 103 |
+
else:
|
| 104 |
+
if prefix_att.shape != (batch, prefix_len):
|
| 105 |
+
raise ValueError(
|
| 106 |
+
f"prefix_att must be shaped {(batch, prefix_len)}, got {tuple(prefix_att.shape)}"
|
| 107 |
+
)
|
| 108 |
+
cum = torch.cumsum(prefix_att.to(device=dev, dtype=torch.long), dim=1)
|
| 109 |
+
prefix_rows = (cum[:, None, :] <= cum[:, :, None]) & p_valid[:, :, None] & p_valid[:, None, :]
|
| 110 |
+
|
| 111 |
+
prefix_pad = torch.zeros((batch, prefix_len, action_len), dtype=torch.bool, device=dev)
|
| 112 |
+
prefix_rows = torch.cat([prefix_rows, prefix_pad], dim=2)
|
| 113 |
+
|
| 114 |
+
nf = prefix_len if non_fast_prefix_len is None else int(non_fast_prefix_len)
|
| 115 |
+
nf = max(0, min(nf, prefix_len))
|
| 116 |
+
action_to_prefix = torch.zeros((batch, action_len, prefix_len), dtype=torch.bool, device=dev)
|
| 117 |
+
if nf > 0:
|
| 118 |
+
action_to_prefix[:, :, :nf] = p_valid[:, None, :nf]
|
| 119 |
+
action_to_prefix &= a_valid[:, :, None]
|
| 120 |
+
|
| 121 |
+
q_block = torch.arange(action_len, device=dev) // int(action_block_size)
|
| 122 |
+
kv_block = q_block
|
| 123 |
+
action_block = q_block[:, None] == kv_block[None, :]
|
| 124 |
+
action_block = action_block[None, :, :].expand(batch, -1, -1)
|
| 125 |
+
action_block = action_block & a_valid[:, :, None] & a_valid[:, None, :]
|
| 126 |
+
action_rows = torch.cat([action_to_prefix, action_block], dim=2)
|
| 127 |
+
|
| 128 |
+
if prefix_rows.shape != (batch, prefix_len, total_len):
|
| 129 |
+
raise AssertionError("internal prefix mask shape error")
|
| 130 |
+
if action_rows.shape != (batch, action_len, total_len):
|
| 131 |
+
raise AssertionError("internal action mask shape error")
|
| 132 |
+
return prefix_rows, action_rows
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
def _bool_to_sdpa_mask(mask: torch.Tensor, q: torch.Tensor) -> torch.Tensor:
|
| 136 |
+
value = MASK_VALUE_F32
|
| 137 |
+
if q.dtype.is_floating_point:
|
| 138 |
+
finfo = torch.finfo(q.dtype)
|
| 139 |
+
value = max(MASK_VALUE_F32, finfo.min)
|
| 140 |
+
return torch.where(
|
| 141 |
+
mask[:, None, :, :],
|
| 142 |
+
torch.zeros((), dtype=q.dtype, device=q.device),
|
| 143 |
+
torch.full((), value, dtype=q.dtype, device=q.device),
|
| 144 |
+
)
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
def _slice_attention_mask(
|
| 148 |
+
attention_mask: torch.Tensor,
|
| 149 |
+
start: int,
|
| 150 |
+
end: int,
|
| 151 |
+
q: torch.Tensor,
|
| 152 |
+
) -> torch.Tensor:
|
| 153 |
+
if attention_mask.dim() == 3:
|
| 154 |
+
mask = attention_mask[:, start:end, :]
|
| 155 |
+
if mask.dtype == torch.bool:
|
| 156 |
+
return mask[:, None, :, :]
|
| 157 |
+
return mask[:, None, :, :].to(dtype=q.dtype)
|
| 158 |
+
if attention_mask.dim() == 4:
|
| 159 |
+
mask = attention_mask[:, :, start:end, :]
|
| 160 |
+
return mask if mask.dtype == torch.bool else mask.to(dtype=q.dtype)
|
| 161 |
+
raise ValueError("attention_mask must be (B, S, S) or (B, 1|H, S, S)")
|
| 162 |
+
|
| 163 |
+
|
| 164 |
+
def _sdpa(
|
| 165 |
+
q: torch.Tensor,
|
| 166 |
+
k: torch.Tensor,
|
| 167 |
+
v: torch.Tensor,
|
| 168 |
+
mask: Optional[torch.Tensor],
|
| 169 |
+
*,
|
| 170 |
+
scale: Optional[float],
|
| 171 |
+
dropout_p: float,
|
| 172 |
+
enable_gqa: bool,
|
| 173 |
+
) -> torch.Tensor:
|
| 174 |
+
kwargs = {"attn_mask": mask, "dropout_p": float(dropout_p), "scale": scale}
|
| 175 |
+
if enable_gqa:
|
| 176 |
+
kwargs["enable_gqa"] = True
|
| 177 |
+
try:
|
| 178 |
+
return F.scaled_dot_product_attention(q, k, v, **kwargs)
|
| 179 |
+
except TypeError:
|
| 180 |
+
kwargs.pop("enable_gqa", None)
|
| 181 |
+
return F.scaled_dot_product_attention(q, k, v, **kwargs)
|
| 182 |
+
|
| 183 |
+
|
| 184 |
+
def reference_flex_attention(
|
| 185 |
+
q: torch.Tensor,
|
| 186 |
+
k: torch.Tensor,
|
| 187 |
+
v: torch.Tensor,
|
| 188 |
+
*,
|
| 189 |
+
prefix_len: int,
|
| 190 |
+
action_block_size: int,
|
| 191 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 192 |
+
prefix_valid: Optional[torch.Tensor] = None,
|
| 193 |
+
prefix_att: Optional[torch.Tensor] = None,
|
| 194 |
+
non_fast_prefix_len: Optional[int] = None,
|
| 195 |
+
action_valid: Optional[torch.Tensor] = None,
|
| 196 |
+
detach_prefix_kv_for_action: bool = True,
|
| 197 |
+
scale: Optional[float] = None,
|
| 198 |
+
dropout_p: float = 0.0,
|
| 199 |
+
enable_gqa: Optional[bool] = None,
|
| 200 |
+
) -> torch.Tensor:
|
| 201 |
+
"""SDPA reference for the PI052 FlexAttention replacement shape.
|
| 202 |
+
|
| 203 |
+
Args:
|
| 204 |
+
q, k, v: ``(B, Hq/Hkv, S, D)`` tensors.
|
| 205 |
+
prefix_len: number of prefix rows/columns at the start of sequence.
|
| 206 |
+
action_block_size: size of each block-diagonal action segment.
|
| 207 |
+
attention_mask: optional prebuilt additive or boolean full mask.
|
| 208 |
+
prefix_valid: optional ``(B, P)`` valid prefix positions.
|
| 209 |
+
prefix_att: optional ``(B, P)`` cumulative-block markers.
|
| 210 |
+
non_fast_prefix_len: prefix columns visible to action rows.
|
| 211 |
+
action_valid: optional ``(B, A)`` valid action positions.
|
| 212 |
+
detach_prefix_kv_for_action: detach prefix K/V on the action-row path.
|
| 213 |
+
scale: SDPA scale. Defaults to ``D ** -0.5``.
|
| 214 |
+
dropout_p: SDPA dropout probability.
|
| 215 |
+
enable_gqa: pass SDPA GQA mode when q heads and kv heads differ.
|
| 216 |
+
"""
|
| 217 |
+
_check_qkv(q, k, v)
|
| 218 |
+
batch, _, total_len, head_dim = q.shape
|
| 219 |
+
if not (0 <= int(prefix_len) <= total_len):
|
| 220 |
+
raise ValueError("prefix_len must be in [0, S]")
|
| 221 |
+
prefix_len = int(prefix_len)
|
| 222 |
+
action_len = total_len - prefix_len
|
| 223 |
+
if scale is None:
|
| 224 |
+
scale = head_dim**-0.5
|
| 225 |
+
if enable_gqa is None:
|
| 226 |
+
enable_gqa = q.shape[1] != k.shape[1]
|
| 227 |
+
|
| 228 |
+
q_prefix = q[:, :, :prefix_len, :]
|
| 229 |
+
q_action = q[:, :, prefix_len:, :]
|
| 230 |
+
k_prefix = k[:, :, :prefix_len, :]
|
| 231 |
+
k_action = k[:, :, prefix_len:, :]
|
| 232 |
+
v_prefix = v[:, :, :prefix_len, :]
|
| 233 |
+
v_action = v[:, :, prefix_len:, :]
|
| 234 |
+
|
| 235 |
+
if attention_mask is None:
|
| 236 |
+
prefix_bool, action_bool = build_block_sparse_bool_masks(
|
| 237 |
+
prefix_valid,
|
| 238 |
+
prefix_att,
|
| 239 |
+
batch=batch,
|
| 240 |
+
prefix_len=prefix_len,
|
| 241 |
+
action_len=action_len,
|
| 242 |
+
action_block_size=action_block_size,
|
| 243 |
+
non_fast_prefix_len=non_fast_prefix_len,
|
| 244 |
+
action_valid=action_valid,
|
| 245 |
+
device=q.device,
|
| 246 |
+
)
|
| 247 |
+
prefix_mask = _bool_to_sdpa_mask(prefix_bool, q)
|
| 248 |
+
action_mask = _bool_to_sdpa_mask(action_bool, q)
|
| 249 |
+
else:
|
| 250 |
+
prefix_mask = _slice_attention_mask(attention_mask, 0, prefix_len, q)
|
| 251 |
+
action_mask = _slice_attention_mask(attention_mask, prefix_len, total_len, q)
|
| 252 |
+
|
| 253 |
+
out_parts = []
|
| 254 |
+
if prefix_len:
|
| 255 |
+
out_parts.append(
|
| 256 |
+
_sdpa(
|
| 257 |
+
q_prefix,
|
| 258 |
+
k,
|
| 259 |
+
v,
|
| 260 |
+
prefix_mask,
|
| 261 |
+
scale=scale,
|
| 262 |
+
dropout_p=dropout_p,
|
| 263 |
+
enable_gqa=bool(enable_gqa),
|
| 264 |
+
)
|
| 265 |
+
)
|
| 266 |
+
if action_len:
|
| 267 |
+
prefix_k = k_prefix.detach() if detach_prefix_kv_for_action else k_prefix
|
| 268 |
+
prefix_v = v_prefix.detach() if detach_prefix_kv_for_action else v_prefix
|
| 269 |
+
k_for_action = torch.cat([prefix_k, k_action], dim=2)
|
| 270 |
+
v_for_action = torch.cat([prefix_v, v_action], dim=2)
|
| 271 |
+
out_parts.append(
|
| 272 |
+
_sdpa(
|
| 273 |
+
q_action,
|
| 274 |
+
k_for_action,
|
| 275 |
+
v_for_action,
|
| 276 |
+
action_mask,
|
| 277 |
+
scale=scale,
|
| 278 |
+
dropout_p=dropout_p,
|
| 279 |
+
enable_gqa=bool(enable_gqa),
|
| 280 |
+
)
|
| 281 |
+
)
|
| 282 |
+
if not out_parts:
|
| 283 |
+
return q.new_empty(q.shape)
|
| 284 |
+
return torch.cat(out_parts, dim=2) if len(out_parts) == 2 else out_parts[0]
|
| 285 |
+
|
| 286 |
+
|
| 287 |
+
def _manual_attention_part(qs, ks, vs, mask, scale):
|
| 288 |
+
"""Materialized-logits attention part: cuBLAS GEMMs + fused masked softmax.
|
| 289 |
+
|
| 290 |
+
Same math as SDPA with an additive mask (fp32 softmax; logits stored in
|
| 291 |
+
the io dtype between the GEMM and the softmax). Grouped queries run as a
|
| 292 |
+
strided batched GEMM over the KV heads, so a 1-head K/V is never
|
| 293 |
+
repeated. At PI052 training shapes (GQA 8:1, D=256, bf16) this beats
|
| 294 |
+
both SDPA-with-dense-mask (2.3-3.1x) and the best FlexAttention
|
| 295 |
+
configuration (1.4-2.9x) on fwd+bwd — see benchmarks/RESULTS.md.
|
| 296 |
+
"""
|
| 297 |
+
B, H, Sq, D = qs.shape
|
| 298 |
+
Hk = ks.shape[1]
|
| 299 |
+
if Hk != H:
|
| 300 |
+
g = H // Hk
|
| 301 |
+
q2 = qs.reshape(B, Hk, g * Sq, D)
|
| 302 |
+
logits = (q2 @ ks.transpose(-1, -2)).reshape(B, H, Sq, -1)
|
| 303 |
+
else:
|
| 304 |
+
logits = qs @ ks.transpose(-1, -2)
|
| 305 |
+
logits = logits * scale
|
| 306 |
+
if mask is not None:
|
| 307 |
+
logits = logits + mask
|
| 308 |
+
p = logits.float().softmax(dim=-1).to(qs.dtype)
|
| 309 |
+
if Hk != H:
|
| 310 |
+
out = (p.reshape(B, Hk, g * Sq, -1) @ vs).reshape(B, H, Sq, D)
|
| 311 |
+
else:
|
| 312 |
+
out = p @ vs
|
| 313 |
+
return out
|
| 314 |
+
|
| 315 |
+
|
| 316 |
+
_manual_part_compiled = None
|
| 317 |
+
|
| 318 |
+
|
| 319 |
+
def _get_manual_part():
|
| 320 |
+
global _manual_part_compiled
|
| 321 |
+
if _manual_part_compiled is None:
|
| 322 |
+
_manual_part_compiled = torch.compile(_manual_attention_part, dynamic=False)
|
| 323 |
+
return _manual_part_compiled
|
| 324 |
+
|
| 325 |
+
|
| 326 |
+
def manual_attention(
|
| 327 |
+
q: torch.Tensor,
|
| 328 |
+
k: torch.Tensor,
|
| 329 |
+
v: torch.Tensor,
|
| 330 |
+
*,
|
| 331 |
+
prefix_len: int,
|
| 332 |
+
action_block_size: int,
|
| 333 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 334 |
+
prefix_valid: Optional[torch.Tensor] = None,
|
| 335 |
+
prefix_att: Optional[torch.Tensor] = None,
|
| 336 |
+
non_fast_prefix_len: Optional[int] = None,
|
| 337 |
+
action_valid: Optional[torch.Tensor] = None,
|
| 338 |
+
detach_prefix_kv_for_action: bool = True,
|
| 339 |
+
scale: Optional[float] = None,
|
| 340 |
+
dropout_p: float = 0.0,
|
| 341 |
+
compile_part: bool = True,
|
| 342 |
+
) -> torch.Tensor:
|
| 343 |
+
"""Materialized-logits implementation of :func:`reference_flex_attention`.
|
| 344 |
+
|
| 345 |
+
Same mask semantics and prefix/action split; each part runs through
|
| 346 |
+
:func:`_manual_attention_part` instead of SDPA. ``dropout_p`` must be 0
|
| 347 |
+
(training attention dropout is unused in PI052); other values raise so
|
| 348 |
+
callers fall back explicitly.
|
| 349 |
+
"""
|
| 350 |
+
if dropout_p:
|
| 351 |
+
raise ValueError("manual_attention does not support dropout; use the reference path")
|
| 352 |
+
_check_qkv(q, k, v)
|
| 353 |
+
batch, _, total_len, head_dim = q.shape
|
| 354 |
+
if not (0 <= int(prefix_len) <= total_len):
|
| 355 |
+
raise ValueError("prefix_len must be in [0, S]")
|
| 356 |
+
prefix_len = int(prefix_len)
|
| 357 |
+
action_len = total_len - prefix_len
|
| 358 |
+
if scale is None:
|
| 359 |
+
scale = head_dim**-0.5
|
| 360 |
+
|
| 361 |
+
if attention_mask is None:
|
| 362 |
+
prefix_bool, action_bool = build_block_sparse_bool_masks(
|
| 363 |
+
prefix_valid,
|
| 364 |
+
prefix_att,
|
| 365 |
+
batch=batch,
|
| 366 |
+
prefix_len=prefix_len,
|
| 367 |
+
action_len=action_len,
|
| 368 |
+
action_block_size=action_block_size,
|
| 369 |
+
non_fast_prefix_len=non_fast_prefix_len,
|
| 370 |
+
action_valid=action_valid,
|
| 371 |
+
device=q.device,
|
| 372 |
+
)
|
| 373 |
+
prefix_mask = _bool_to_sdpa_mask(prefix_bool, q)
|
| 374 |
+
action_mask = _bool_to_sdpa_mask(action_bool, q)
|
| 375 |
+
else:
|
| 376 |
+
prefix_mask = _slice_attention_mask(attention_mask, 0, prefix_len, q)
|
| 377 |
+
action_mask = _slice_attention_mask(attention_mask, prefix_len, total_len, q)
|
| 378 |
+
if prefix_mask.dtype == torch.bool:
|
| 379 |
+
prefix_mask = _bool_to_sdpa_mask(prefix_mask[:, 0], q)
|
| 380 |
+
if action_mask.dtype == torch.bool:
|
| 381 |
+
action_mask = _bool_to_sdpa_mask(action_mask[:, 0], q)
|
| 382 |
+
|
| 383 |
+
part = _get_manual_part() if compile_part else _manual_attention_part
|
| 384 |
+
out_parts = []
|
| 385 |
+
if prefix_len:
|
| 386 |
+
out_parts.append(part(q[:, :, :prefix_len, :], k, v, prefix_mask, scale))
|
| 387 |
+
if action_len:
|
| 388 |
+
k_prefix = k[:, :, :prefix_len, :]
|
| 389 |
+
v_prefix = v[:, :, :prefix_len, :]
|
| 390 |
+
if detach_prefix_kv_for_action:
|
| 391 |
+
k_prefix = k_prefix.detach()
|
| 392 |
+
v_prefix = v_prefix.detach()
|
| 393 |
+
k_for_action = torch.cat([k_prefix, k[:, :, prefix_len:, :]], dim=2)
|
| 394 |
+
v_for_action = torch.cat([v_prefix, v[:, :, prefix_len:, :]], dim=2)
|
| 395 |
+
out_parts.append(part(q[:, :, prefix_len:, :], k_for_action, v_for_action, action_mask, scale))
|
| 396 |
+
if not out_parts:
|
| 397 |
+
return q.new_empty(q.shape)
|
| 398 |
+
return torch.cat(out_parts, dim=2) if len(out_parts) == 2 else out_parts[0]
|
| 399 |
+
|
| 400 |
+
|
| 401 |
+
def flex_attention(
|
| 402 |
+
q: torch.Tensor,
|
| 403 |
+
k: torch.Tensor,
|
| 404 |
+
v: torch.Tensor,
|
| 405 |
+
*,
|
| 406 |
+
prefix_len: int,
|
| 407 |
+
action_block_size: int,
|
| 408 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 409 |
+
prefix_valid: Optional[torch.Tensor] = None,
|
| 410 |
+
prefix_att: Optional[torch.Tensor] = None,
|
| 411 |
+
non_fast_prefix_len: Optional[int] = None,
|
| 412 |
+
action_valid: Optional[torch.Tensor] = None,
|
| 413 |
+
detach_prefix_kv_for_action: bool = True,
|
| 414 |
+
scale: Optional[float] = None,
|
| 415 |
+
dropout_p: float = 0.0,
|
| 416 |
+
enable_gqa: Optional[bool] = None,
|
| 417 |
+
force_fallback: bool = False,
|
| 418 |
+
impl: str = "sdpa",
|
| 419 |
+
) -> torch.Tensor:
|
| 420 |
+
"""Flex-style block-sparse attention.
|
| 421 |
+
|
| 422 |
+
``impl="sdpa"`` (default) keeps the SDPA reference path;
|
| 423 |
+
``impl="manual"`` routes through the materialized-logits
|
| 424 |
+
implementation; ``impl="auto"`` picks manual only where it has been
|
| 425 |
+
measured to win end-to-end — consumer Blackwell (sm120-class) with
|
| 426 |
+
no dropout. On A100 (sm80) the manual math wins microbenches but
|
| 427 |
+
loses training-step integration, and on H100/H200 (sm90) the fused
|
| 428 |
+
FMHA kernels win outright, so auto keeps SDPA there.
|
| 429 |
+
"""
|
| 430 |
+
_ = force_fallback
|
| 431 |
+
if impl == "auto":
|
| 432 |
+
sm120 = q.is_cuda and torch.cuda.get_device_capability(q.device)[0] == 12
|
| 433 |
+
impl = "manual" if (sm120 and not dropout_p) else "sdpa"
|
| 434 |
+
if impl == "manual":
|
| 435 |
+
return manual_attention(
|
| 436 |
+
q,
|
| 437 |
+
k,
|
| 438 |
+
v,
|
| 439 |
+
prefix_len=prefix_len,
|
| 440 |
+
action_block_size=action_block_size,
|
| 441 |
+
attention_mask=attention_mask,
|
| 442 |
+
prefix_valid=prefix_valid,
|
| 443 |
+
prefix_att=prefix_att,
|
| 444 |
+
non_fast_prefix_len=non_fast_prefix_len,
|
| 445 |
+
action_valid=action_valid,
|
| 446 |
+
detach_prefix_kv_for_action=detach_prefix_kv_for_action,
|
| 447 |
+
scale=scale,
|
| 448 |
+
dropout_p=dropout_p,
|
| 449 |
+
)
|
| 450 |
+
return reference_flex_attention(
|
| 451 |
+
q,
|
| 452 |
+
k,
|
| 453 |
+
v,
|
| 454 |
+
prefix_len=prefix_len,
|
| 455 |
+
action_block_size=action_block_size,
|
| 456 |
+
attention_mask=attention_mask,
|
| 457 |
+
prefix_valid=prefix_valid,
|
| 458 |
+
prefix_att=prefix_att,
|
| 459 |
+
non_fast_prefix_len=non_fast_prefix_len,
|
| 460 |
+
action_valid=action_valid,
|
| 461 |
+
detach_prefix_kv_for_action=detach_prefix_kv_for_action,
|
| 462 |
+
scale=scale,
|
| 463 |
+
dropout_p=dropout_p,
|
| 464 |
+
enable_gqa=enable_gqa,
|
| 465 |
+
)
|
| 466 |
+
|
| 467 |
+
|
| 468 |
+
def flex_attention_forward(*args, **kwargs) -> torch.Tensor:
|
| 469 |
+
"""Forward-only compatibility wrapper."""
|
| 470 |
+
return flex_attention(*args, **kwargs)
|
| 471 |
+
|
| 472 |
+
|
| 473 |
+
def backend_marker(x: torch.Tensor) -> torch.Tensor:
|
| 474 |
+
if ops is None:
|
| 475 |
+
return x
|
| 476 |
+
return ops._flashrt_training_package_marker(x)
|
| 477 |
+
|
| 478 |
+
|
| 479 |
+
__all__ = [
|
| 480 |
+
"MASK_VALUE_F32",
|
| 481 |
+
"backend_marker",
|
| 482 |
+
"build_block_sparse_bool_masks",
|
| 483 |
+
"flex_attention",
|
| 484 |
+
"flex_attention_forward",
|
| 485 |
+
"manual_attention",
|
| 486 |
+
"reference_flex_attention",
|
| 487 |
+
]
|
build/torch211-cxx11-cu128-x86_64-linux/_flashrt_flex_attention_train_cuda_eaf7454.abi3.so
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0c55e2302f1031c5b9071ba00e373d2ef6f63f3b5f2c2c3cab80cf91806a6ed5
|
| 3 |
+
size 52792
|
build/torch211-cxx11-cu128-x86_64-linux/_ops.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
from . import _flashrt_flex_attention_train_cuda_eaf7454
|
| 3 |
+
ops = torch.ops._flashrt_flex_attention_train_cuda_eaf7454
|
| 4 |
+
|
| 5 |
+
def add_op_namespace_prefix(op_name: str):
|
| 6 |
+
"""
|
| 7 |
+
Prefix op by namespace.
|
| 8 |
+
"""
|
| 9 |
+
return f"_flashrt_flex_attention_train_cuda_eaf7454::{op_name}"
|
build/torch211-cxx11-cu128-x86_64-linux/flashrt_flex_attention_train/__init__.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import ctypes
|
| 2 |
+
import importlib.util
|
| 3 |
+
import sys
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
from types import ModuleType
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def _import_from_path(file_path: Path) -> ModuleType:
|
| 9 |
+
# We cannot use the module name as-is, after adding it to `sys.modules`,
|
| 10 |
+
# it would also be used for other imports. So, we make a module name that
|
| 11 |
+
# depends on the path for it to be unique using the hex-encoded hash of
|
| 12 |
+
# the path.
|
| 13 |
+
path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
|
| 14 |
+
module_name = path_hash
|
| 15 |
+
spec = importlib.util.spec_from_file_location(module_name, file_path)
|
| 16 |
+
if spec is None:
|
| 17 |
+
raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
|
| 18 |
+
module = importlib.util.module_from_spec(spec)
|
| 19 |
+
if module is None:
|
| 20 |
+
raise ImportError(f"Cannot load module {module_name} from spec")
|
| 21 |
+
sys.modules[module_name] = module
|
| 22 |
+
spec.loader.exec_module(module) # type: ignore
|
| 23 |
+
return module
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
|
build/torch211-cxx11-cu128-x86_64-linux/metadata.json
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "flashrt-flex-attention-train",
|
| 3 |
+
"id": "_flashrt_flex_attention_train_cuda_eaf7454",
|
| 4 |
+
"version": 1,
|
| 5 |
+
"license": "Apache-2.0",
|
| 6 |
+
"python-depends": [],
|
| 7 |
+
"backend": {
|
| 8 |
+
"type": "cuda",
|
| 9 |
+
"archs": [
|
| 10 |
+
"10.0",
|
| 11 |
+
"10.1",
|
| 12 |
+
"12.0+PTX",
|
| 13 |
+
"7.0",
|
| 14 |
+
"7.2",
|
| 15 |
+
"7.5",
|
| 16 |
+
"8.0",
|
| 17 |
+
"8.6",
|
| 18 |
+
"8.7",
|
| 19 |
+
"8.9",
|
| 20 |
+
"9.0"
|
| 21 |
+
]
|
| 22 |
+
}
|
| 23 |
+
}
|
build/torch211-cxx11-cu130-x86_64-linux/__init__.py
ADDED
|
@@ -0,0 +1,487 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""FlashRT Flex-style block-sparse attention training API.
|
| 2 |
+
|
| 3 |
+
The public function implements the PI052 prefix/action mask pattern:
|
| 4 |
+
|
| 5 |
+
* prefix query rows use the original K/V tensors, so prefix losses keep normal
|
| 6 |
+
gradients into prefix K/V;
|
| 7 |
+
* action query rows read detached prefix K/V plus normal action K/V by default,
|
| 8 |
+
matching the current training semantics.
|
| 9 |
+
|
| 10 |
+
Unsupported shapes route to the SDPA reference path. Native CUDA kernels are
|
| 11 |
+
not exposed until a shape-specialized implementation beats SDPA on the target
|
| 12 |
+
A100/5090 validation matrix.
|
| 13 |
+
"""
|
| 14 |
+
|
| 15 |
+
from __future__ import annotations
|
| 16 |
+
|
| 17 |
+
from typing import Optional
|
| 18 |
+
|
| 19 |
+
import torch
|
| 20 |
+
import torch.nn.functional as F
|
| 21 |
+
|
| 22 |
+
try:
|
| 23 |
+
from ._ops import ops
|
| 24 |
+
|
| 25 |
+
_HAS_OPS = hasattr(ops, "_flashrt_training_package_marker")
|
| 26 |
+
except Exception: # source-tree tests before kernel-builder creates _ops.py
|
| 27 |
+
ops = None
|
| 28 |
+
_HAS_OPS = False
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
MASK_VALUE_F32 = -2.3819763e38
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def _use_ops(namespace_ops) -> None:
|
| 35 |
+
"""Install a manually built extension (dev/testing path)."""
|
| 36 |
+
global ops, _HAS_OPS
|
| 37 |
+
ops = namespace_ops
|
| 38 |
+
_HAS_OPS = hasattr(ops, "_flashrt_training_package_marker")
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def _check_qkv(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor) -> None:
|
| 42 |
+
if q.dim() != 4 or k.dim() != 4 or v.dim() != 4:
|
| 43 |
+
raise ValueError("q, k, and v must be shaped (B, H, S, D)")
|
| 44 |
+
if q.shape[0] != k.shape[0] or q.shape[0] != v.shape[0]:
|
| 45 |
+
raise ValueError("q, k, and v batch dimensions must match")
|
| 46 |
+
if k.shape != v.shape:
|
| 47 |
+
raise ValueError("k and v shapes must match")
|
| 48 |
+
if q.shape[2] != k.shape[2] or q.shape[3] != k.shape[3]:
|
| 49 |
+
raise ValueError("q, k, and v sequence/head_dim dimensions must match")
|
| 50 |
+
if q.device != k.device or q.device != v.device:
|
| 51 |
+
raise ValueError("q, k, and v must be on the same device")
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def _as_valid(mask: Optional[torch.Tensor], batch: int, length: int, device: torch.device) -> torch.Tensor:
|
| 55 |
+
if mask is None:
|
| 56 |
+
return torch.ones((batch, length), dtype=torch.bool, device=device)
|
| 57 |
+
if mask.shape != (batch, length):
|
| 58 |
+
raise ValueError(f"mask must be shaped {(batch, length)}, got {tuple(mask.shape)}")
|
| 59 |
+
return mask.to(device=device, dtype=torch.bool)
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def build_block_sparse_bool_masks(
|
| 63 |
+
prefix_valid: Optional[torch.Tensor],
|
| 64 |
+
prefix_att: Optional[torch.Tensor],
|
| 65 |
+
*,
|
| 66 |
+
batch: int,
|
| 67 |
+
prefix_len: int,
|
| 68 |
+
action_len: int,
|
| 69 |
+
action_block_size: int,
|
| 70 |
+
non_fast_prefix_len: Optional[int] = None,
|
| 71 |
+
action_valid: Optional[torch.Tensor] = None,
|
| 72 |
+
device: Optional[torch.device] = None,
|
| 73 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 74 |
+
"""Build boolean masks for the split FlexAttention SDPA calls.
|
| 75 |
+
|
| 76 |
+
Returns ``(prefix_rows, action_rows)`` with shapes ``(B, P, S)`` and
|
| 77 |
+
``(B, A, S)``. Boolean True means the key/value position is visible.
|
| 78 |
+
|
| 79 |
+
``prefix_att`` follows Lerobot's cumulative-block convention: prefix key
|
| 80 |
+
``j`` is visible to prefix query ``i`` when ``cumsum(prefix_att)[j] <=
|
| 81 |
+
cumsum(prefix_att)[i]`` and both rows are valid. When omitted, prefix rows
|
| 82 |
+
attend to all valid prefix tokens.
|
| 83 |
+
"""
|
| 84 |
+
if action_block_size <= 0:
|
| 85 |
+
raise ValueError("action_block_size must be positive")
|
| 86 |
+
if prefix_len < 0 or action_len < 0:
|
| 87 |
+
raise ValueError("prefix_len and action_len must be non-negative")
|
| 88 |
+
total_len = prefix_len + action_len
|
| 89 |
+
dev = device
|
| 90 |
+
if dev is None:
|
| 91 |
+
for t in (prefix_valid, prefix_att, action_valid):
|
| 92 |
+
if t is not None:
|
| 93 |
+
dev = t.device
|
| 94 |
+
break
|
| 95 |
+
if dev is None:
|
| 96 |
+
dev = torch.device("cpu")
|
| 97 |
+
|
| 98 |
+
p_valid = _as_valid(prefix_valid, batch, prefix_len, dev)
|
| 99 |
+
a_valid = _as_valid(action_valid, batch, action_len, dev)
|
| 100 |
+
|
| 101 |
+
if prefix_att is None:
|
| 102 |
+
prefix_rows = p_valid[:, :, None] & p_valid[:, None, :]
|
| 103 |
+
else:
|
| 104 |
+
if prefix_att.shape != (batch, prefix_len):
|
| 105 |
+
raise ValueError(
|
| 106 |
+
f"prefix_att must be shaped {(batch, prefix_len)}, got {tuple(prefix_att.shape)}"
|
| 107 |
+
)
|
| 108 |
+
cum = torch.cumsum(prefix_att.to(device=dev, dtype=torch.long), dim=1)
|
| 109 |
+
prefix_rows = (cum[:, None, :] <= cum[:, :, None]) & p_valid[:, :, None] & p_valid[:, None, :]
|
| 110 |
+
|
| 111 |
+
prefix_pad = torch.zeros((batch, prefix_len, action_len), dtype=torch.bool, device=dev)
|
| 112 |
+
prefix_rows = torch.cat([prefix_rows, prefix_pad], dim=2)
|
| 113 |
+
|
| 114 |
+
nf = prefix_len if non_fast_prefix_len is None else int(non_fast_prefix_len)
|
| 115 |
+
nf = max(0, min(nf, prefix_len))
|
| 116 |
+
action_to_prefix = torch.zeros((batch, action_len, prefix_len), dtype=torch.bool, device=dev)
|
| 117 |
+
if nf > 0:
|
| 118 |
+
action_to_prefix[:, :, :nf] = p_valid[:, None, :nf]
|
| 119 |
+
action_to_prefix &= a_valid[:, :, None]
|
| 120 |
+
|
| 121 |
+
q_block = torch.arange(action_len, device=dev) // int(action_block_size)
|
| 122 |
+
kv_block = q_block
|
| 123 |
+
action_block = q_block[:, None] == kv_block[None, :]
|
| 124 |
+
action_block = action_block[None, :, :].expand(batch, -1, -1)
|
| 125 |
+
action_block = action_block & a_valid[:, :, None] & a_valid[:, None, :]
|
| 126 |
+
action_rows = torch.cat([action_to_prefix, action_block], dim=2)
|
| 127 |
+
|
| 128 |
+
if prefix_rows.shape != (batch, prefix_len, total_len):
|
| 129 |
+
raise AssertionError("internal prefix mask shape error")
|
| 130 |
+
if action_rows.shape != (batch, action_len, total_len):
|
| 131 |
+
raise AssertionError("internal action mask shape error")
|
| 132 |
+
return prefix_rows, action_rows
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
def _bool_to_sdpa_mask(mask: torch.Tensor, q: torch.Tensor) -> torch.Tensor:
|
| 136 |
+
value = MASK_VALUE_F32
|
| 137 |
+
if q.dtype.is_floating_point:
|
| 138 |
+
finfo = torch.finfo(q.dtype)
|
| 139 |
+
value = max(MASK_VALUE_F32, finfo.min)
|
| 140 |
+
return torch.where(
|
| 141 |
+
mask[:, None, :, :],
|
| 142 |
+
torch.zeros((), dtype=q.dtype, device=q.device),
|
| 143 |
+
torch.full((), value, dtype=q.dtype, device=q.device),
|
| 144 |
+
)
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
def _slice_attention_mask(
|
| 148 |
+
attention_mask: torch.Tensor,
|
| 149 |
+
start: int,
|
| 150 |
+
end: int,
|
| 151 |
+
q: torch.Tensor,
|
| 152 |
+
) -> torch.Tensor:
|
| 153 |
+
if attention_mask.dim() == 3:
|
| 154 |
+
mask = attention_mask[:, start:end, :]
|
| 155 |
+
if mask.dtype == torch.bool:
|
| 156 |
+
return mask[:, None, :, :]
|
| 157 |
+
return mask[:, None, :, :].to(dtype=q.dtype)
|
| 158 |
+
if attention_mask.dim() == 4:
|
| 159 |
+
mask = attention_mask[:, :, start:end, :]
|
| 160 |
+
return mask if mask.dtype == torch.bool else mask.to(dtype=q.dtype)
|
| 161 |
+
raise ValueError("attention_mask must be (B, S, S) or (B, 1|H, S, S)")
|
| 162 |
+
|
| 163 |
+
|
| 164 |
+
def _sdpa(
|
| 165 |
+
q: torch.Tensor,
|
| 166 |
+
k: torch.Tensor,
|
| 167 |
+
v: torch.Tensor,
|
| 168 |
+
mask: Optional[torch.Tensor],
|
| 169 |
+
*,
|
| 170 |
+
scale: Optional[float],
|
| 171 |
+
dropout_p: float,
|
| 172 |
+
enable_gqa: bool,
|
| 173 |
+
) -> torch.Tensor:
|
| 174 |
+
kwargs = {"attn_mask": mask, "dropout_p": float(dropout_p), "scale": scale}
|
| 175 |
+
if enable_gqa:
|
| 176 |
+
kwargs["enable_gqa"] = True
|
| 177 |
+
try:
|
| 178 |
+
return F.scaled_dot_product_attention(q, k, v, **kwargs)
|
| 179 |
+
except TypeError:
|
| 180 |
+
kwargs.pop("enable_gqa", None)
|
| 181 |
+
return F.scaled_dot_product_attention(q, k, v, **kwargs)
|
| 182 |
+
|
| 183 |
+
|
| 184 |
+
def reference_flex_attention(
|
| 185 |
+
q: torch.Tensor,
|
| 186 |
+
k: torch.Tensor,
|
| 187 |
+
v: torch.Tensor,
|
| 188 |
+
*,
|
| 189 |
+
prefix_len: int,
|
| 190 |
+
action_block_size: int,
|
| 191 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 192 |
+
prefix_valid: Optional[torch.Tensor] = None,
|
| 193 |
+
prefix_att: Optional[torch.Tensor] = None,
|
| 194 |
+
non_fast_prefix_len: Optional[int] = None,
|
| 195 |
+
action_valid: Optional[torch.Tensor] = None,
|
| 196 |
+
detach_prefix_kv_for_action: bool = True,
|
| 197 |
+
scale: Optional[float] = None,
|
| 198 |
+
dropout_p: float = 0.0,
|
| 199 |
+
enable_gqa: Optional[bool] = None,
|
| 200 |
+
) -> torch.Tensor:
|
| 201 |
+
"""SDPA reference for the PI052 FlexAttention replacement shape.
|
| 202 |
+
|
| 203 |
+
Args:
|
| 204 |
+
q, k, v: ``(B, Hq/Hkv, S, D)`` tensors.
|
| 205 |
+
prefix_len: number of prefix rows/columns at the start of sequence.
|
| 206 |
+
action_block_size: size of each block-diagonal action segment.
|
| 207 |
+
attention_mask: optional prebuilt additive or boolean full mask.
|
| 208 |
+
prefix_valid: optional ``(B, P)`` valid prefix positions.
|
| 209 |
+
prefix_att: optional ``(B, P)`` cumulative-block markers.
|
| 210 |
+
non_fast_prefix_len: prefix columns visible to action rows.
|
| 211 |
+
action_valid: optional ``(B, A)`` valid action positions.
|
| 212 |
+
detach_prefix_kv_for_action: detach prefix K/V on the action-row path.
|
| 213 |
+
scale: SDPA scale. Defaults to ``D ** -0.5``.
|
| 214 |
+
dropout_p: SDPA dropout probability.
|
| 215 |
+
enable_gqa: pass SDPA GQA mode when q heads and kv heads differ.
|
| 216 |
+
"""
|
| 217 |
+
_check_qkv(q, k, v)
|
| 218 |
+
batch, _, total_len, head_dim = q.shape
|
| 219 |
+
if not (0 <= int(prefix_len) <= total_len):
|
| 220 |
+
raise ValueError("prefix_len must be in [0, S]")
|
| 221 |
+
prefix_len = int(prefix_len)
|
| 222 |
+
action_len = total_len - prefix_len
|
| 223 |
+
if scale is None:
|
| 224 |
+
scale = head_dim**-0.5
|
| 225 |
+
if enable_gqa is None:
|
| 226 |
+
enable_gqa = q.shape[1] != k.shape[1]
|
| 227 |
+
|
| 228 |
+
q_prefix = q[:, :, :prefix_len, :]
|
| 229 |
+
q_action = q[:, :, prefix_len:, :]
|
| 230 |
+
k_prefix = k[:, :, :prefix_len, :]
|
| 231 |
+
k_action = k[:, :, prefix_len:, :]
|
| 232 |
+
v_prefix = v[:, :, :prefix_len, :]
|
| 233 |
+
v_action = v[:, :, prefix_len:, :]
|
| 234 |
+
|
| 235 |
+
if attention_mask is None:
|
| 236 |
+
prefix_bool, action_bool = build_block_sparse_bool_masks(
|
| 237 |
+
prefix_valid,
|
| 238 |
+
prefix_att,
|
| 239 |
+
batch=batch,
|
| 240 |
+
prefix_len=prefix_len,
|
| 241 |
+
action_len=action_len,
|
| 242 |
+
action_block_size=action_block_size,
|
| 243 |
+
non_fast_prefix_len=non_fast_prefix_len,
|
| 244 |
+
action_valid=action_valid,
|
| 245 |
+
device=q.device,
|
| 246 |
+
)
|
| 247 |
+
prefix_mask = _bool_to_sdpa_mask(prefix_bool, q)
|
| 248 |
+
action_mask = _bool_to_sdpa_mask(action_bool, q)
|
| 249 |
+
else:
|
| 250 |
+
prefix_mask = _slice_attention_mask(attention_mask, 0, prefix_len, q)
|
| 251 |
+
action_mask = _slice_attention_mask(attention_mask, prefix_len, total_len, q)
|
| 252 |
+
|
| 253 |
+
out_parts = []
|
| 254 |
+
if prefix_len:
|
| 255 |
+
out_parts.append(
|
| 256 |
+
_sdpa(
|
| 257 |
+
q_prefix,
|
| 258 |
+
k,
|
| 259 |
+
v,
|
| 260 |
+
prefix_mask,
|
| 261 |
+
scale=scale,
|
| 262 |
+
dropout_p=dropout_p,
|
| 263 |
+
enable_gqa=bool(enable_gqa),
|
| 264 |
+
)
|
| 265 |
+
)
|
| 266 |
+
if action_len:
|
| 267 |
+
prefix_k = k_prefix.detach() if detach_prefix_kv_for_action else k_prefix
|
| 268 |
+
prefix_v = v_prefix.detach() if detach_prefix_kv_for_action else v_prefix
|
| 269 |
+
k_for_action = torch.cat([prefix_k, k_action], dim=2)
|
| 270 |
+
v_for_action = torch.cat([prefix_v, v_action], dim=2)
|
| 271 |
+
out_parts.append(
|
| 272 |
+
_sdpa(
|
| 273 |
+
q_action,
|
| 274 |
+
k_for_action,
|
| 275 |
+
v_for_action,
|
| 276 |
+
action_mask,
|
| 277 |
+
scale=scale,
|
| 278 |
+
dropout_p=dropout_p,
|
| 279 |
+
enable_gqa=bool(enable_gqa),
|
| 280 |
+
)
|
| 281 |
+
)
|
| 282 |
+
if not out_parts:
|
| 283 |
+
return q.new_empty(q.shape)
|
| 284 |
+
return torch.cat(out_parts, dim=2) if len(out_parts) == 2 else out_parts[0]
|
| 285 |
+
|
| 286 |
+
|
| 287 |
+
def _manual_attention_part(qs, ks, vs, mask, scale):
|
| 288 |
+
"""Materialized-logits attention part: cuBLAS GEMMs + fused masked softmax.
|
| 289 |
+
|
| 290 |
+
Same math as SDPA with an additive mask (fp32 softmax; logits stored in
|
| 291 |
+
the io dtype between the GEMM and the softmax). Grouped queries run as a
|
| 292 |
+
strided batched GEMM over the KV heads, so a 1-head K/V is never
|
| 293 |
+
repeated. At PI052 training shapes (GQA 8:1, D=256, bf16) this beats
|
| 294 |
+
both SDPA-with-dense-mask (2.3-3.1x) and the best FlexAttention
|
| 295 |
+
configuration (1.4-2.9x) on fwd+bwd — see benchmarks/RESULTS.md.
|
| 296 |
+
"""
|
| 297 |
+
B, H, Sq, D = qs.shape
|
| 298 |
+
Hk = ks.shape[1]
|
| 299 |
+
if Hk != H:
|
| 300 |
+
g = H // Hk
|
| 301 |
+
q2 = qs.reshape(B, Hk, g * Sq, D)
|
| 302 |
+
logits = (q2 @ ks.transpose(-1, -2)).reshape(B, H, Sq, -1)
|
| 303 |
+
else:
|
| 304 |
+
logits = qs @ ks.transpose(-1, -2)
|
| 305 |
+
logits = logits * scale
|
| 306 |
+
if mask is not None:
|
| 307 |
+
logits = logits + mask
|
| 308 |
+
p = logits.float().softmax(dim=-1).to(qs.dtype)
|
| 309 |
+
if Hk != H:
|
| 310 |
+
out = (p.reshape(B, Hk, g * Sq, -1) @ vs).reshape(B, H, Sq, D)
|
| 311 |
+
else:
|
| 312 |
+
out = p @ vs
|
| 313 |
+
return out
|
| 314 |
+
|
| 315 |
+
|
| 316 |
+
_manual_part_compiled = None
|
| 317 |
+
|
| 318 |
+
|
| 319 |
+
def _get_manual_part():
|
| 320 |
+
global _manual_part_compiled
|
| 321 |
+
if _manual_part_compiled is None:
|
| 322 |
+
_manual_part_compiled = torch.compile(_manual_attention_part, dynamic=False)
|
| 323 |
+
return _manual_part_compiled
|
| 324 |
+
|
| 325 |
+
|
| 326 |
+
def manual_attention(
|
| 327 |
+
q: torch.Tensor,
|
| 328 |
+
k: torch.Tensor,
|
| 329 |
+
v: torch.Tensor,
|
| 330 |
+
*,
|
| 331 |
+
prefix_len: int,
|
| 332 |
+
action_block_size: int,
|
| 333 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 334 |
+
prefix_valid: Optional[torch.Tensor] = None,
|
| 335 |
+
prefix_att: Optional[torch.Tensor] = None,
|
| 336 |
+
non_fast_prefix_len: Optional[int] = None,
|
| 337 |
+
action_valid: Optional[torch.Tensor] = None,
|
| 338 |
+
detach_prefix_kv_for_action: bool = True,
|
| 339 |
+
scale: Optional[float] = None,
|
| 340 |
+
dropout_p: float = 0.0,
|
| 341 |
+
compile_part: bool = True,
|
| 342 |
+
) -> torch.Tensor:
|
| 343 |
+
"""Materialized-logits implementation of :func:`reference_flex_attention`.
|
| 344 |
+
|
| 345 |
+
Same mask semantics and prefix/action split; each part runs through
|
| 346 |
+
:func:`_manual_attention_part` instead of SDPA. ``dropout_p`` must be 0
|
| 347 |
+
(training attention dropout is unused in PI052); other values raise so
|
| 348 |
+
callers fall back explicitly.
|
| 349 |
+
"""
|
| 350 |
+
if dropout_p:
|
| 351 |
+
raise ValueError("manual_attention does not support dropout; use the reference path")
|
| 352 |
+
_check_qkv(q, k, v)
|
| 353 |
+
batch, _, total_len, head_dim = q.shape
|
| 354 |
+
if not (0 <= int(prefix_len) <= total_len):
|
| 355 |
+
raise ValueError("prefix_len must be in [0, S]")
|
| 356 |
+
prefix_len = int(prefix_len)
|
| 357 |
+
action_len = total_len - prefix_len
|
| 358 |
+
if scale is None:
|
| 359 |
+
scale = head_dim**-0.5
|
| 360 |
+
|
| 361 |
+
if attention_mask is None:
|
| 362 |
+
prefix_bool, action_bool = build_block_sparse_bool_masks(
|
| 363 |
+
prefix_valid,
|
| 364 |
+
prefix_att,
|
| 365 |
+
batch=batch,
|
| 366 |
+
prefix_len=prefix_len,
|
| 367 |
+
action_len=action_len,
|
| 368 |
+
action_block_size=action_block_size,
|
| 369 |
+
non_fast_prefix_len=non_fast_prefix_len,
|
| 370 |
+
action_valid=action_valid,
|
| 371 |
+
device=q.device,
|
| 372 |
+
)
|
| 373 |
+
prefix_mask = _bool_to_sdpa_mask(prefix_bool, q)
|
| 374 |
+
action_mask = _bool_to_sdpa_mask(action_bool, q)
|
| 375 |
+
else:
|
| 376 |
+
prefix_mask = _slice_attention_mask(attention_mask, 0, prefix_len, q)
|
| 377 |
+
action_mask = _slice_attention_mask(attention_mask, prefix_len, total_len, q)
|
| 378 |
+
if prefix_mask.dtype == torch.bool:
|
| 379 |
+
prefix_mask = _bool_to_sdpa_mask(prefix_mask[:, 0], q)
|
| 380 |
+
if action_mask.dtype == torch.bool:
|
| 381 |
+
action_mask = _bool_to_sdpa_mask(action_mask[:, 0], q)
|
| 382 |
+
|
| 383 |
+
part = _get_manual_part() if compile_part else _manual_attention_part
|
| 384 |
+
out_parts = []
|
| 385 |
+
if prefix_len:
|
| 386 |
+
out_parts.append(part(q[:, :, :prefix_len, :], k, v, prefix_mask, scale))
|
| 387 |
+
if action_len:
|
| 388 |
+
k_prefix = k[:, :, :prefix_len, :]
|
| 389 |
+
v_prefix = v[:, :, :prefix_len, :]
|
| 390 |
+
if detach_prefix_kv_for_action:
|
| 391 |
+
k_prefix = k_prefix.detach()
|
| 392 |
+
v_prefix = v_prefix.detach()
|
| 393 |
+
k_for_action = torch.cat([k_prefix, k[:, :, prefix_len:, :]], dim=2)
|
| 394 |
+
v_for_action = torch.cat([v_prefix, v[:, :, prefix_len:, :]], dim=2)
|
| 395 |
+
out_parts.append(part(q[:, :, prefix_len:, :], k_for_action, v_for_action, action_mask, scale))
|
| 396 |
+
if not out_parts:
|
| 397 |
+
return q.new_empty(q.shape)
|
| 398 |
+
return torch.cat(out_parts, dim=2) if len(out_parts) == 2 else out_parts[0]
|
| 399 |
+
|
| 400 |
+
|
| 401 |
+
def flex_attention(
|
| 402 |
+
q: torch.Tensor,
|
| 403 |
+
k: torch.Tensor,
|
| 404 |
+
v: torch.Tensor,
|
| 405 |
+
*,
|
| 406 |
+
prefix_len: int,
|
| 407 |
+
action_block_size: int,
|
| 408 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 409 |
+
prefix_valid: Optional[torch.Tensor] = None,
|
| 410 |
+
prefix_att: Optional[torch.Tensor] = None,
|
| 411 |
+
non_fast_prefix_len: Optional[int] = None,
|
| 412 |
+
action_valid: Optional[torch.Tensor] = None,
|
| 413 |
+
detach_prefix_kv_for_action: bool = True,
|
| 414 |
+
scale: Optional[float] = None,
|
| 415 |
+
dropout_p: float = 0.0,
|
| 416 |
+
enable_gqa: Optional[bool] = None,
|
| 417 |
+
force_fallback: bool = False,
|
| 418 |
+
impl: str = "sdpa",
|
| 419 |
+
) -> torch.Tensor:
|
| 420 |
+
"""Flex-style block-sparse attention.
|
| 421 |
+
|
| 422 |
+
``impl="sdpa"`` (default) keeps the SDPA reference path;
|
| 423 |
+
``impl="manual"`` routes through the materialized-logits
|
| 424 |
+
implementation; ``impl="auto"`` picks manual only where it has been
|
| 425 |
+
measured to win end-to-end — consumer Blackwell (sm120-class) with
|
| 426 |
+
no dropout. On A100 (sm80) the manual math wins microbenches but
|
| 427 |
+
loses training-step integration, and on H100/H200 (sm90) the fused
|
| 428 |
+
FMHA kernels win outright, so auto keeps SDPA there.
|
| 429 |
+
"""
|
| 430 |
+
_ = force_fallback
|
| 431 |
+
if impl == "auto":
|
| 432 |
+
sm120 = q.is_cuda and torch.cuda.get_device_capability(q.device)[0] == 12
|
| 433 |
+
impl = "manual" if (sm120 and not dropout_p) else "sdpa"
|
| 434 |
+
if impl == "manual":
|
| 435 |
+
return manual_attention(
|
| 436 |
+
q,
|
| 437 |
+
k,
|
| 438 |
+
v,
|
| 439 |
+
prefix_len=prefix_len,
|
| 440 |
+
action_block_size=action_block_size,
|
| 441 |
+
attention_mask=attention_mask,
|
| 442 |
+
prefix_valid=prefix_valid,
|
| 443 |
+
prefix_att=prefix_att,
|
| 444 |
+
non_fast_prefix_len=non_fast_prefix_len,
|
| 445 |
+
action_valid=action_valid,
|
| 446 |
+
detach_prefix_kv_for_action=detach_prefix_kv_for_action,
|
| 447 |
+
scale=scale,
|
| 448 |
+
dropout_p=dropout_p,
|
| 449 |
+
)
|
| 450 |
+
return reference_flex_attention(
|
| 451 |
+
q,
|
| 452 |
+
k,
|
| 453 |
+
v,
|
| 454 |
+
prefix_len=prefix_len,
|
| 455 |
+
action_block_size=action_block_size,
|
| 456 |
+
attention_mask=attention_mask,
|
| 457 |
+
prefix_valid=prefix_valid,
|
| 458 |
+
prefix_att=prefix_att,
|
| 459 |
+
non_fast_prefix_len=non_fast_prefix_len,
|
| 460 |
+
action_valid=action_valid,
|
| 461 |
+
detach_prefix_kv_for_action=detach_prefix_kv_for_action,
|
| 462 |
+
scale=scale,
|
| 463 |
+
dropout_p=dropout_p,
|
| 464 |
+
enable_gqa=enable_gqa,
|
| 465 |
+
)
|
| 466 |
+
|
| 467 |
+
|
| 468 |
+
def flex_attention_forward(*args, **kwargs) -> torch.Tensor:
|
| 469 |
+
"""Forward-only compatibility wrapper."""
|
| 470 |
+
return flex_attention(*args, **kwargs)
|
| 471 |
+
|
| 472 |
+
|
| 473 |
+
def backend_marker(x: torch.Tensor) -> torch.Tensor:
|
| 474 |
+
if ops is None:
|
| 475 |
+
return x
|
| 476 |
+
return ops._flashrt_training_package_marker(x)
|
| 477 |
+
|
| 478 |
+
|
| 479 |
+
__all__ = [
|
| 480 |
+
"MASK_VALUE_F32",
|
| 481 |
+
"backend_marker",
|
| 482 |
+
"build_block_sparse_bool_masks",
|
| 483 |
+
"flex_attention",
|
| 484 |
+
"flex_attention_forward",
|
| 485 |
+
"manual_attention",
|
| 486 |
+
"reference_flex_attention",
|
| 487 |
+
]
|
build/torch211-cxx11-cu130-x86_64-linux/_flashrt_flex_attention_train_cuda_eaf7454.abi3.so
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:9aa4c1d74ad4357d787307b8980e4cd4e0641b45d1b41e3d8285955f1eeb4d0c
|
| 3 |
+
size 56888
|
build/torch211-cxx11-cu130-x86_64-linux/_ops.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
from . import _flashrt_flex_attention_train_cuda_eaf7454
|
| 3 |
+
ops = torch.ops._flashrt_flex_attention_train_cuda_eaf7454
|
| 4 |
+
|
| 5 |
+
def add_op_namespace_prefix(op_name: str):
|
| 6 |
+
"""
|
| 7 |
+
Prefix op by namespace.
|
| 8 |
+
"""
|
| 9 |
+
return f"_flashrt_flex_attention_train_cuda_eaf7454::{op_name}"
|
build/torch211-cxx11-cu130-x86_64-linux/flashrt_flex_attention_train/__init__.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import ctypes
|
| 2 |
+
import importlib.util
|
| 3 |
+
import sys
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
from types import ModuleType
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def _import_from_path(file_path: Path) -> ModuleType:
|
| 9 |
+
# We cannot use the module name as-is, after adding it to `sys.modules`,
|
| 10 |
+
# it would also be used for other imports. So, we make a module name that
|
| 11 |
+
# depends on the path for it to be unique using the hex-encoded hash of
|
| 12 |
+
# the path.
|
| 13 |
+
path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
|
| 14 |
+
module_name = path_hash
|
| 15 |
+
spec = importlib.util.spec_from_file_location(module_name, file_path)
|
| 16 |
+
if spec is None:
|
| 17 |
+
raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
|
| 18 |
+
module = importlib.util.module_from_spec(spec)
|
| 19 |
+
if module is None:
|
| 20 |
+
raise ImportError(f"Cannot load module {module_name} from spec")
|
| 21 |
+
sys.modules[module_name] = module
|
| 22 |
+
spec.loader.exec_module(module) # type: ignore
|
| 23 |
+
return module
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
|
build/torch211-cxx11-cu130-x86_64-linux/metadata.json
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "flashrt-flex-attention-train",
|
| 3 |
+
"id": "_flashrt_flex_attention_train_cuda_eaf7454",
|
| 4 |
+
"version": 1,
|
| 5 |
+
"license": "Apache-2.0",
|
| 6 |
+
"python-depends": [],
|
| 7 |
+
"backend": {
|
| 8 |
+
"type": "cuda",
|
| 9 |
+
"archs": [
|
| 10 |
+
"10.0",
|
| 11 |
+
"11.0",
|
| 12 |
+
"12.0",
|
| 13 |
+
"12.1+PTX",
|
| 14 |
+
"7.5",
|
| 15 |
+
"8.0",
|
| 16 |
+
"8.6",
|
| 17 |
+
"8.7",
|
| 18 |
+
"8.9",
|
| 19 |
+
"9.0"
|
| 20 |
+
]
|
| 21 |
+
}
|
| 22 |
+
}
|
build/torch212-cxx11-cu130-x86_64-linux/__init__.py
ADDED
|
@@ -0,0 +1,487 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""FlashRT Flex-style block-sparse attention training API.
|
| 2 |
+
|
| 3 |
+
The public function implements the PI052 prefix/action mask pattern:
|
| 4 |
+
|
| 5 |
+
* prefix query rows use the original K/V tensors, so prefix losses keep normal
|
| 6 |
+
gradients into prefix K/V;
|
| 7 |
+
* action query rows read detached prefix K/V plus normal action K/V by default,
|
| 8 |
+
matching the current training semantics.
|
| 9 |
+
|
| 10 |
+
Unsupported shapes route to the SDPA reference path. Native CUDA kernels are
|
| 11 |
+
not exposed until a shape-specialized implementation beats SDPA on the target
|
| 12 |
+
A100/5090 validation matrix.
|
| 13 |
+
"""
|
| 14 |
+
|
| 15 |
+
from __future__ import annotations
|
| 16 |
+
|
| 17 |
+
from typing import Optional
|
| 18 |
+
|
| 19 |
+
import torch
|
| 20 |
+
import torch.nn.functional as F
|
| 21 |
+
|
| 22 |
+
try:
|
| 23 |
+
from ._ops import ops
|
| 24 |
+
|
| 25 |
+
_HAS_OPS = hasattr(ops, "_flashrt_training_package_marker")
|
| 26 |
+
except Exception: # source-tree tests before kernel-builder creates _ops.py
|
| 27 |
+
ops = None
|
| 28 |
+
_HAS_OPS = False
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
MASK_VALUE_F32 = -2.3819763e38
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def _use_ops(namespace_ops) -> None:
|
| 35 |
+
"""Install a manually built extension (dev/testing path)."""
|
| 36 |
+
global ops, _HAS_OPS
|
| 37 |
+
ops = namespace_ops
|
| 38 |
+
_HAS_OPS = hasattr(ops, "_flashrt_training_package_marker")
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def _check_qkv(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor) -> None:
|
| 42 |
+
if q.dim() != 4 or k.dim() != 4 or v.dim() != 4:
|
| 43 |
+
raise ValueError("q, k, and v must be shaped (B, H, S, D)")
|
| 44 |
+
if q.shape[0] != k.shape[0] or q.shape[0] != v.shape[0]:
|
| 45 |
+
raise ValueError("q, k, and v batch dimensions must match")
|
| 46 |
+
if k.shape != v.shape:
|
| 47 |
+
raise ValueError("k and v shapes must match")
|
| 48 |
+
if q.shape[2] != k.shape[2] or q.shape[3] != k.shape[3]:
|
| 49 |
+
raise ValueError("q, k, and v sequence/head_dim dimensions must match")
|
| 50 |
+
if q.device != k.device or q.device != v.device:
|
| 51 |
+
raise ValueError("q, k, and v must be on the same device")
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def _as_valid(mask: Optional[torch.Tensor], batch: int, length: int, device: torch.device) -> torch.Tensor:
|
| 55 |
+
if mask is None:
|
| 56 |
+
return torch.ones((batch, length), dtype=torch.bool, device=device)
|
| 57 |
+
if mask.shape != (batch, length):
|
| 58 |
+
raise ValueError(f"mask must be shaped {(batch, length)}, got {tuple(mask.shape)}")
|
| 59 |
+
return mask.to(device=device, dtype=torch.bool)
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def build_block_sparse_bool_masks(
|
| 63 |
+
prefix_valid: Optional[torch.Tensor],
|
| 64 |
+
prefix_att: Optional[torch.Tensor],
|
| 65 |
+
*,
|
| 66 |
+
batch: int,
|
| 67 |
+
prefix_len: int,
|
| 68 |
+
action_len: int,
|
| 69 |
+
action_block_size: int,
|
| 70 |
+
non_fast_prefix_len: Optional[int] = None,
|
| 71 |
+
action_valid: Optional[torch.Tensor] = None,
|
| 72 |
+
device: Optional[torch.device] = None,
|
| 73 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 74 |
+
"""Build boolean masks for the split FlexAttention SDPA calls.
|
| 75 |
+
|
| 76 |
+
Returns ``(prefix_rows, action_rows)`` with shapes ``(B, P, S)`` and
|
| 77 |
+
``(B, A, S)``. Boolean True means the key/value position is visible.
|
| 78 |
+
|
| 79 |
+
``prefix_att`` follows Lerobot's cumulative-block convention: prefix key
|
| 80 |
+
``j`` is visible to prefix query ``i`` when ``cumsum(prefix_att)[j] <=
|
| 81 |
+
cumsum(prefix_att)[i]`` and both rows are valid. When omitted, prefix rows
|
| 82 |
+
attend to all valid prefix tokens.
|
| 83 |
+
"""
|
| 84 |
+
if action_block_size <= 0:
|
| 85 |
+
raise ValueError("action_block_size must be positive")
|
| 86 |
+
if prefix_len < 0 or action_len < 0:
|
| 87 |
+
raise ValueError("prefix_len and action_len must be non-negative")
|
| 88 |
+
total_len = prefix_len + action_len
|
| 89 |
+
dev = device
|
| 90 |
+
if dev is None:
|
| 91 |
+
for t in (prefix_valid, prefix_att, action_valid):
|
| 92 |
+
if t is not None:
|
| 93 |
+
dev = t.device
|
| 94 |
+
break
|
| 95 |
+
if dev is None:
|
| 96 |
+
dev = torch.device("cpu")
|
| 97 |
+
|
| 98 |
+
p_valid = _as_valid(prefix_valid, batch, prefix_len, dev)
|
| 99 |
+
a_valid = _as_valid(action_valid, batch, action_len, dev)
|
| 100 |
+
|
| 101 |
+
if prefix_att is None:
|
| 102 |
+
prefix_rows = p_valid[:, :, None] & p_valid[:, None, :]
|
| 103 |
+
else:
|
| 104 |
+
if prefix_att.shape != (batch, prefix_len):
|
| 105 |
+
raise ValueError(
|
| 106 |
+
f"prefix_att must be shaped {(batch, prefix_len)}, got {tuple(prefix_att.shape)}"
|
| 107 |
+
)
|
| 108 |
+
cum = torch.cumsum(prefix_att.to(device=dev, dtype=torch.long), dim=1)
|
| 109 |
+
prefix_rows = (cum[:, None, :] <= cum[:, :, None]) & p_valid[:, :, None] & p_valid[:, None, :]
|
| 110 |
+
|
| 111 |
+
prefix_pad = torch.zeros((batch, prefix_len, action_len), dtype=torch.bool, device=dev)
|
| 112 |
+
prefix_rows = torch.cat([prefix_rows, prefix_pad], dim=2)
|
| 113 |
+
|
| 114 |
+
nf = prefix_len if non_fast_prefix_len is None else int(non_fast_prefix_len)
|
| 115 |
+
nf = max(0, min(nf, prefix_len))
|
| 116 |
+
action_to_prefix = torch.zeros((batch, action_len, prefix_len), dtype=torch.bool, device=dev)
|
| 117 |
+
if nf > 0:
|
| 118 |
+
action_to_prefix[:, :, :nf] = p_valid[:, None, :nf]
|
| 119 |
+
action_to_prefix &= a_valid[:, :, None]
|
| 120 |
+
|
| 121 |
+
q_block = torch.arange(action_len, device=dev) // int(action_block_size)
|
| 122 |
+
kv_block = q_block
|
| 123 |
+
action_block = q_block[:, None] == kv_block[None, :]
|
| 124 |
+
action_block = action_block[None, :, :].expand(batch, -1, -1)
|
| 125 |
+
action_block = action_block & a_valid[:, :, None] & a_valid[:, None, :]
|
| 126 |
+
action_rows = torch.cat([action_to_prefix, action_block], dim=2)
|
| 127 |
+
|
| 128 |
+
if prefix_rows.shape != (batch, prefix_len, total_len):
|
| 129 |
+
raise AssertionError("internal prefix mask shape error")
|
| 130 |
+
if action_rows.shape != (batch, action_len, total_len):
|
| 131 |
+
raise AssertionError("internal action mask shape error")
|
| 132 |
+
return prefix_rows, action_rows
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
def _bool_to_sdpa_mask(mask: torch.Tensor, q: torch.Tensor) -> torch.Tensor:
|
| 136 |
+
value = MASK_VALUE_F32
|
| 137 |
+
if q.dtype.is_floating_point:
|
| 138 |
+
finfo = torch.finfo(q.dtype)
|
| 139 |
+
value = max(MASK_VALUE_F32, finfo.min)
|
| 140 |
+
return torch.where(
|
| 141 |
+
mask[:, None, :, :],
|
| 142 |
+
torch.zeros((), dtype=q.dtype, device=q.device),
|
| 143 |
+
torch.full((), value, dtype=q.dtype, device=q.device),
|
| 144 |
+
)
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
def _slice_attention_mask(
|
| 148 |
+
attention_mask: torch.Tensor,
|
| 149 |
+
start: int,
|
| 150 |
+
end: int,
|
| 151 |
+
q: torch.Tensor,
|
| 152 |
+
) -> torch.Tensor:
|
| 153 |
+
if attention_mask.dim() == 3:
|
| 154 |
+
mask = attention_mask[:, start:end, :]
|
| 155 |
+
if mask.dtype == torch.bool:
|
| 156 |
+
return mask[:, None, :, :]
|
| 157 |
+
return mask[:, None, :, :].to(dtype=q.dtype)
|
| 158 |
+
if attention_mask.dim() == 4:
|
| 159 |
+
mask = attention_mask[:, :, start:end, :]
|
| 160 |
+
return mask if mask.dtype == torch.bool else mask.to(dtype=q.dtype)
|
| 161 |
+
raise ValueError("attention_mask must be (B, S, S) or (B, 1|H, S, S)")
|
| 162 |
+
|
| 163 |
+
|
| 164 |
+
def _sdpa(
|
| 165 |
+
q: torch.Tensor,
|
| 166 |
+
k: torch.Tensor,
|
| 167 |
+
v: torch.Tensor,
|
| 168 |
+
mask: Optional[torch.Tensor],
|
| 169 |
+
*,
|
| 170 |
+
scale: Optional[float],
|
| 171 |
+
dropout_p: float,
|
| 172 |
+
enable_gqa: bool,
|
| 173 |
+
) -> torch.Tensor:
|
| 174 |
+
kwargs = {"attn_mask": mask, "dropout_p": float(dropout_p), "scale": scale}
|
| 175 |
+
if enable_gqa:
|
| 176 |
+
kwargs["enable_gqa"] = True
|
| 177 |
+
try:
|
| 178 |
+
return F.scaled_dot_product_attention(q, k, v, **kwargs)
|
| 179 |
+
except TypeError:
|
| 180 |
+
kwargs.pop("enable_gqa", None)
|
| 181 |
+
return F.scaled_dot_product_attention(q, k, v, **kwargs)
|
| 182 |
+
|
| 183 |
+
|
| 184 |
+
def reference_flex_attention(
|
| 185 |
+
q: torch.Tensor,
|
| 186 |
+
k: torch.Tensor,
|
| 187 |
+
v: torch.Tensor,
|
| 188 |
+
*,
|
| 189 |
+
prefix_len: int,
|
| 190 |
+
action_block_size: int,
|
| 191 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 192 |
+
prefix_valid: Optional[torch.Tensor] = None,
|
| 193 |
+
prefix_att: Optional[torch.Tensor] = None,
|
| 194 |
+
non_fast_prefix_len: Optional[int] = None,
|
| 195 |
+
action_valid: Optional[torch.Tensor] = None,
|
| 196 |
+
detach_prefix_kv_for_action: bool = True,
|
| 197 |
+
scale: Optional[float] = None,
|
| 198 |
+
dropout_p: float = 0.0,
|
| 199 |
+
enable_gqa: Optional[bool] = None,
|
| 200 |
+
) -> torch.Tensor:
|
| 201 |
+
"""SDPA reference for the PI052 FlexAttention replacement shape.
|
| 202 |
+
|
| 203 |
+
Args:
|
| 204 |
+
q, k, v: ``(B, Hq/Hkv, S, D)`` tensors.
|
| 205 |
+
prefix_len: number of prefix rows/columns at the start of sequence.
|
| 206 |
+
action_block_size: size of each block-diagonal action segment.
|
| 207 |
+
attention_mask: optional prebuilt additive or boolean full mask.
|
| 208 |
+
prefix_valid: optional ``(B, P)`` valid prefix positions.
|
| 209 |
+
prefix_att: optional ``(B, P)`` cumulative-block markers.
|
| 210 |
+
non_fast_prefix_len: prefix columns visible to action rows.
|
| 211 |
+
action_valid: optional ``(B, A)`` valid action positions.
|
| 212 |
+
detach_prefix_kv_for_action: detach prefix K/V on the action-row path.
|
| 213 |
+
scale: SDPA scale. Defaults to ``D ** -0.5``.
|
| 214 |
+
dropout_p: SDPA dropout probability.
|
| 215 |
+
enable_gqa: pass SDPA GQA mode when q heads and kv heads differ.
|
| 216 |
+
"""
|
| 217 |
+
_check_qkv(q, k, v)
|
| 218 |
+
batch, _, total_len, head_dim = q.shape
|
| 219 |
+
if not (0 <= int(prefix_len) <= total_len):
|
| 220 |
+
raise ValueError("prefix_len must be in [0, S]")
|
| 221 |
+
prefix_len = int(prefix_len)
|
| 222 |
+
action_len = total_len - prefix_len
|
| 223 |
+
if scale is None:
|
| 224 |
+
scale = head_dim**-0.5
|
| 225 |
+
if enable_gqa is None:
|
| 226 |
+
enable_gqa = q.shape[1] != k.shape[1]
|
| 227 |
+
|
| 228 |
+
q_prefix = q[:, :, :prefix_len, :]
|
| 229 |
+
q_action = q[:, :, prefix_len:, :]
|
| 230 |
+
k_prefix = k[:, :, :prefix_len, :]
|
| 231 |
+
k_action = k[:, :, prefix_len:, :]
|
| 232 |
+
v_prefix = v[:, :, :prefix_len, :]
|
| 233 |
+
v_action = v[:, :, prefix_len:, :]
|
| 234 |
+
|
| 235 |
+
if attention_mask is None:
|
| 236 |
+
prefix_bool, action_bool = build_block_sparse_bool_masks(
|
| 237 |
+
prefix_valid,
|
| 238 |
+
prefix_att,
|
| 239 |
+
batch=batch,
|
| 240 |
+
prefix_len=prefix_len,
|
| 241 |
+
action_len=action_len,
|
| 242 |
+
action_block_size=action_block_size,
|
| 243 |
+
non_fast_prefix_len=non_fast_prefix_len,
|
| 244 |
+
action_valid=action_valid,
|
| 245 |
+
device=q.device,
|
| 246 |
+
)
|
| 247 |
+
prefix_mask = _bool_to_sdpa_mask(prefix_bool, q)
|
| 248 |
+
action_mask = _bool_to_sdpa_mask(action_bool, q)
|
| 249 |
+
else:
|
| 250 |
+
prefix_mask = _slice_attention_mask(attention_mask, 0, prefix_len, q)
|
| 251 |
+
action_mask = _slice_attention_mask(attention_mask, prefix_len, total_len, q)
|
| 252 |
+
|
| 253 |
+
out_parts = []
|
| 254 |
+
if prefix_len:
|
| 255 |
+
out_parts.append(
|
| 256 |
+
_sdpa(
|
| 257 |
+
q_prefix,
|
| 258 |
+
k,
|
| 259 |
+
v,
|
| 260 |
+
prefix_mask,
|
| 261 |
+
scale=scale,
|
| 262 |
+
dropout_p=dropout_p,
|
| 263 |
+
enable_gqa=bool(enable_gqa),
|
| 264 |
+
)
|
| 265 |
+
)
|
| 266 |
+
if action_len:
|
| 267 |
+
prefix_k = k_prefix.detach() if detach_prefix_kv_for_action else k_prefix
|
| 268 |
+
prefix_v = v_prefix.detach() if detach_prefix_kv_for_action else v_prefix
|
| 269 |
+
k_for_action = torch.cat([prefix_k, k_action], dim=2)
|
| 270 |
+
v_for_action = torch.cat([prefix_v, v_action], dim=2)
|
| 271 |
+
out_parts.append(
|
| 272 |
+
_sdpa(
|
| 273 |
+
q_action,
|
| 274 |
+
k_for_action,
|
| 275 |
+
v_for_action,
|
| 276 |
+
action_mask,
|
| 277 |
+
scale=scale,
|
| 278 |
+
dropout_p=dropout_p,
|
| 279 |
+
enable_gqa=bool(enable_gqa),
|
| 280 |
+
)
|
| 281 |
+
)
|
| 282 |
+
if not out_parts:
|
| 283 |
+
return q.new_empty(q.shape)
|
| 284 |
+
return torch.cat(out_parts, dim=2) if len(out_parts) == 2 else out_parts[0]
|
| 285 |
+
|
| 286 |
+
|
| 287 |
+
def _manual_attention_part(qs, ks, vs, mask, scale):
|
| 288 |
+
"""Materialized-logits attention part: cuBLAS GEMMs + fused masked softmax.
|
| 289 |
+
|
| 290 |
+
Same math as SDPA with an additive mask (fp32 softmax; logits stored in
|
| 291 |
+
the io dtype between the GEMM and the softmax). Grouped queries run as a
|
| 292 |
+
strided batched GEMM over the KV heads, so a 1-head K/V is never
|
| 293 |
+
repeated. At PI052 training shapes (GQA 8:1, D=256, bf16) this beats
|
| 294 |
+
both SDPA-with-dense-mask (2.3-3.1x) and the best FlexAttention
|
| 295 |
+
configuration (1.4-2.9x) on fwd+bwd — see benchmarks/RESULTS.md.
|
| 296 |
+
"""
|
| 297 |
+
B, H, Sq, D = qs.shape
|
| 298 |
+
Hk = ks.shape[1]
|
| 299 |
+
if Hk != H:
|
| 300 |
+
g = H // Hk
|
| 301 |
+
q2 = qs.reshape(B, Hk, g * Sq, D)
|
| 302 |
+
logits = (q2 @ ks.transpose(-1, -2)).reshape(B, H, Sq, -1)
|
| 303 |
+
else:
|
| 304 |
+
logits = qs @ ks.transpose(-1, -2)
|
| 305 |
+
logits = logits * scale
|
| 306 |
+
if mask is not None:
|
| 307 |
+
logits = logits + mask
|
| 308 |
+
p = logits.float().softmax(dim=-1).to(qs.dtype)
|
| 309 |
+
if Hk != H:
|
| 310 |
+
out = (p.reshape(B, Hk, g * Sq, -1) @ vs).reshape(B, H, Sq, D)
|
| 311 |
+
else:
|
| 312 |
+
out = p @ vs
|
| 313 |
+
return out
|
| 314 |
+
|
| 315 |
+
|
| 316 |
+
_manual_part_compiled = None
|
| 317 |
+
|
| 318 |
+
|
| 319 |
+
def _get_manual_part():
|
| 320 |
+
global _manual_part_compiled
|
| 321 |
+
if _manual_part_compiled is None:
|
| 322 |
+
_manual_part_compiled = torch.compile(_manual_attention_part, dynamic=False)
|
| 323 |
+
return _manual_part_compiled
|
| 324 |
+
|
| 325 |
+
|
| 326 |
+
def manual_attention(
|
| 327 |
+
q: torch.Tensor,
|
| 328 |
+
k: torch.Tensor,
|
| 329 |
+
v: torch.Tensor,
|
| 330 |
+
*,
|
| 331 |
+
prefix_len: int,
|
| 332 |
+
action_block_size: int,
|
| 333 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 334 |
+
prefix_valid: Optional[torch.Tensor] = None,
|
| 335 |
+
prefix_att: Optional[torch.Tensor] = None,
|
| 336 |
+
non_fast_prefix_len: Optional[int] = None,
|
| 337 |
+
action_valid: Optional[torch.Tensor] = None,
|
| 338 |
+
detach_prefix_kv_for_action: bool = True,
|
| 339 |
+
scale: Optional[float] = None,
|
| 340 |
+
dropout_p: float = 0.0,
|
| 341 |
+
compile_part: bool = True,
|
| 342 |
+
) -> torch.Tensor:
|
| 343 |
+
"""Materialized-logits implementation of :func:`reference_flex_attention`.
|
| 344 |
+
|
| 345 |
+
Same mask semantics and prefix/action split; each part runs through
|
| 346 |
+
:func:`_manual_attention_part` instead of SDPA. ``dropout_p`` must be 0
|
| 347 |
+
(training attention dropout is unused in PI052); other values raise so
|
| 348 |
+
callers fall back explicitly.
|
| 349 |
+
"""
|
| 350 |
+
if dropout_p:
|
| 351 |
+
raise ValueError("manual_attention does not support dropout; use the reference path")
|
| 352 |
+
_check_qkv(q, k, v)
|
| 353 |
+
batch, _, total_len, head_dim = q.shape
|
| 354 |
+
if not (0 <= int(prefix_len) <= total_len):
|
| 355 |
+
raise ValueError("prefix_len must be in [0, S]")
|
| 356 |
+
prefix_len = int(prefix_len)
|
| 357 |
+
action_len = total_len - prefix_len
|
| 358 |
+
if scale is None:
|
| 359 |
+
scale = head_dim**-0.5
|
| 360 |
+
|
| 361 |
+
if attention_mask is None:
|
| 362 |
+
prefix_bool, action_bool = build_block_sparse_bool_masks(
|
| 363 |
+
prefix_valid,
|
| 364 |
+
prefix_att,
|
| 365 |
+
batch=batch,
|
| 366 |
+
prefix_len=prefix_len,
|
| 367 |
+
action_len=action_len,
|
| 368 |
+
action_block_size=action_block_size,
|
| 369 |
+
non_fast_prefix_len=non_fast_prefix_len,
|
| 370 |
+
action_valid=action_valid,
|
| 371 |
+
device=q.device,
|
| 372 |
+
)
|
| 373 |
+
prefix_mask = _bool_to_sdpa_mask(prefix_bool, q)
|
| 374 |
+
action_mask = _bool_to_sdpa_mask(action_bool, q)
|
| 375 |
+
else:
|
| 376 |
+
prefix_mask = _slice_attention_mask(attention_mask, 0, prefix_len, q)
|
| 377 |
+
action_mask = _slice_attention_mask(attention_mask, prefix_len, total_len, q)
|
| 378 |
+
if prefix_mask.dtype == torch.bool:
|
| 379 |
+
prefix_mask = _bool_to_sdpa_mask(prefix_mask[:, 0], q)
|
| 380 |
+
if action_mask.dtype == torch.bool:
|
| 381 |
+
action_mask = _bool_to_sdpa_mask(action_mask[:, 0], q)
|
| 382 |
+
|
| 383 |
+
part = _get_manual_part() if compile_part else _manual_attention_part
|
| 384 |
+
out_parts = []
|
| 385 |
+
if prefix_len:
|
| 386 |
+
out_parts.append(part(q[:, :, :prefix_len, :], k, v, prefix_mask, scale))
|
| 387 |
+
if action_len:
|
| 388 |
+
k_prefix = k[:, :, :prefix_len, :]
|
| 389 |
+
v_prefix = v[:, :, :prefix_len, :]
|
| 390 |
+
if detach_prefix_kv_for_action:
|
| 391 |
+
k_prefix = k_prefix.detach()
|
| 392 |
+
v_prefix = v_prefix.detach()
|
| 393 |
+
k_for_action = torch.cat([k_prefix, k[:, :, prefix_len:, :]], dim=2)
|
| 394 |
+
v_for_action = torch.cat([v_prefix, v[:, :, prefix_len:, :]], dim=2)
|
| 395 |
+
out_parts.append(part(q[:, :, prefix_len:, :], k_for_action, v_for_action, action_mask, scale))
|
| 396 |
+
if not out_parts:
|
| 397 |
+
return q.new_empty(q.shape)
|
| 398 |
+
return torch.cat(out_parts, dim=2) if len(out_parts) == 2 else out_parts[0]
|
| 399 |
+
|
| 400 |
+
|
| 401 |
+
def flex_attention(
|
| 402 |
+
q: torch.Tensor,
|
| 403 |
+
k: torch.Tensor,
|
| 404 |
+
v: torch.Tensor,
|
| 405 |
+
*,
|
| 406 |
+
prefix_len: int,
|
| 407 |
+
action_block_size: int,
|
| 408 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 409 |
+
prefix_valid: Optional[torch.Tensor] = None,
|
| 410 |
+
prefix_att: Optional[torch.Tensor] = None,
|
| 411 |
+
non_fast_prefix_len: Optional[int] = None,
|
| 412 |
+
action_valid: Optional[torch.Tensor] = None,
|
| 413 |
+
detach_prefix_kv_for_action: bool = True,
|
| 414 |
+
scale: Optional[float] = None,
|
| 415 |
+
dropout_p: float = 0.0,
|
| 416 |
+
enable_gqa: Optional[bool] = None,
|
| 417 |
+
force_fallback: bool = False,
|
| 418 |
+
impl: str = "sdpa",
|
| 419 |
+
) -> torch.Tensor:
|
| 420 |
+
"""Flex-style block-sparse attention.
|
| 421 |
+
|
| 422 |
+
``impl="sdpa"`` (default) keeps the SDPA reference path;
|
| 423 |
+
``impl="manual"`` routes through the materialized-logits
|
| 424 |
+
implementation; ``impl="auto"`` picks manual only where it has been
|
| 425 |
+
measured to win end-to-end — consumer Blackwell (sm120-class) with
|
| 426 |
+
no dropout. On A100 (sm80) the manual math wins microbenches but
|
| 427 |
+
loses training-step integration, and on H100/H200 (sm90) the fused
|
| 428 |
+
FMHA kernels win outright, so auto keeps SDPA there.
|
| 429 |
+
"""
|
| 430 |
+
_ = force_fallback
|
| 431 |
+
if impl == "auto":
|
| 432 |
+
sm120 = q.is_cuda and torch.cuda.get_device_capability(q.device)[0] == 12
|
| 433 |
+
impl = "manual" if (sm120 and not dropout_p) else "sdpa"
|
| 434 |
+
if impl == "manual":
|
| 435 |
+
return manual_attention(
|
| 436 |
+
q,
|
| 437 |
+
k,
|
| 438 |
+
v,
|
| 439 |
+
prefix_len=prefix_len,
|
| 440 |
+
action_block_size=action_block_size,
|
| 441 |
+
attention_mask=attention_mask,
|
| 442 |
+
prefix_valid=prefix_valid,
|
| 443 |
+
prefix_att=prefix_att,
|
| 444 |
+
non_fast_prefix_len=non_fast_prefix_len,
|
| 445 |
+
action_valid=action_valid,
|
| 446 |
+
detach_prefix_kv_for_action=detach_prefix_kv_for_action,
|
| 447 |
+
scale=scale,
|
| 448 |
+
dropout_p=dropout_p,
|
| 449 |
+
)
|
| 450 |
+
return reference_flex_attention(
|
| 451 |
+
q,
|
| 452 |
+
k,
|
| 453 |
+
v,
|
| 454 |
+
prefix_len=prefix_len,
|
| 455 |
+
action_block_size=action_block_size,
|
| 456 |
+
attention_mask=attention_mask,
|
| 457 |
+
prefix_valid=prefix_valid,
|
| 458 |
+
prefix_att=prefix_att,
|
| 459 |
+
non_fast_prefix_len=non_fast_prefix_len,
|
| 460 |
+
action_valid=action_valid,
|
| 461 |
+
detach_prefix_kv_for_action=detach_prefix_kv_for_action,
|
| 462 |
+
scale=scale,
|
| 463 |
+
dropout_p=dropout_p,
|
| 464 |
+
enable_gqa=enable_gqa,
|
| 465 |
+
)
|
| 466 |
+
|
| 467 |
+
|
| 468 |
+
def flex_attention_forward(*args, **kwargs) -> torch.Tensor:
|
| 469 |
+
"""Forward-only compatibility wrapper."""
|
| 470 |
+
return flex_attention(*args, **kwargs)
|
| 471 |
+
|
| 472 |
+
|
| 473 |
+
def backend_marker(x: torch.Tensor) -> torch.Tensor:
|
| 474 |
+
if ops is None:
|
| 475 |
+
return x
|
| 476 |
+
return ops._flashrt_training_package_marker(x)
|
| 477 |
+
|
| 478 |
+
|
| 479 |
+
__all__ = [
|
| 480 |
+
"MASK_VALUE_F32",
|
| 481 |
+
"backend_marker",
|
| 482 |
+
"build_block_sparse_bool_masks",
|
| 483 |
+
"flex_attention",
|
| 484 |
+
"flex_attention_forward",
|
| 485 |
+
"manual_attention",
|
| 486 |
+
"reference_flex_attention",
|
| 487 |
+
]
|
build/torch212-cxx11-cu130-x86_64-linux/_flashrt_flex_attention_train_cuda_eaf7454.abi3.so
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:6afaa6ed896df3e147d469f64c9c08fde09d265c3c6a8a07489f97abfdbb5b77
|
| 3 |
+
size 56800
|
build/torch212-cxx11-cu130-x86_64-linux/_ops.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
from . import _flashrt_flex_attention_train_cuda_eaf7454
|
| 3 |
+
ops = torch.ops._flashrt_flex_attention_train_cuda_eaf7454
|
| 4 |
+
|
| 5 |
+
def add_op_namespace_prefix(op_name: str):
|
| 6 |
+
"""
|
| 7 |
+
Prefix op by namespace.
|
| 8 |
+
"""
|
| 9 |
+
return f"_flashrt_flex_attention_train_cuda_eaf7454::{op_name}"
|
build/torch212-cxx11-cu130-x86_64-linux/flashrt_flex_attention_train/__init__.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import ctypes
|
| 2 |
+
import importlib.util
|
| 3 |
+
import sys
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
from types import ModuleType
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def _import_from_path(file_path: Path) -> ModuleType:
|
| 9 |
+
# We cannot use the module name as-is, after adding it to `sys.modules`,
|
| 10 |
+
# it would also be used for other imports. So, we make a module name that
|
| 11 |
+
# depends on the path for it to be unique using the hex-encoded hash of
|
| 12 |
+
# the path.
|
| 13 |
+
path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
|
| 14 |
+
module_name = path_hash
|
| 15 |
+
spec = importlib.util.spec_from_file_location(module_name, file_path)
|
| 16 |
+
if spec is None:
|
| 17 |
+
raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
|
| 18 |
+
module = importlib.util.module_from_spec(spec)
|
| 19 |
+
if module is None:
|
| 20 |
+
raise ImportError(f"Cannot load module {module_name} from spec")
|
| 21 |
+
sys.modules[module_name] = module
|
| 22 |
+
spec.loader.exec_module(module) # type: ignore
|
| 23 |
+
return module
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
|
build/torch212-cxx11-cu130-x86_64-linux/metadata.json
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "flashrt-flex-attention-train",
|
| 3 |
+
"id": "_flashrt_flex_attention_train_cuda_eaf7454",
|
| 4 |
+
"version": 1,
|
| 5 |
+
"license": "Apache-2.0",
|
| 6 |
+
"python-depends": [],
|
| 7 |
+
"backend": {
|
| 8 |
+
"type": "cuda",
|
| 9 |
+
"archs": [
|
| 10 |
+
"10.0",
|
| 11 |
+
"11.0",
|
| 12 |
+
"12.0",
|
| 13 |
+
"12.1+PTX",
|
| 14 |
+
"7.5",
|
| 15 |
+
"8.0",
|
| 16 |
+
"8.6",
|
| 17 |
+
"8.7",
|
| 18 |
+
"8.9",
|
| 19 |
+
"9.0"
|
| 20 |
+
]
|
| 21 |
+
}
|
| 22 |
+
}
|
build/torch212-cxx11-cu132-x86_64-linux/__init__.py
ADDED
|
@@ -0,0 +1,487 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""FlashRT Flex-style block-sparse attention training API.
|
| 2 |
+
|
| 3 |
+
The public function implements the PI052 prefix/action mask pattern:
|
| 4 |
+
|
| 5 |
+
* prefix query rows use the original K/V tensors, so prefix losses keep normal
|
| 6 |
+
gradients into prefix K/V;
|
| 7 |
+
* action query rows read detached prefix K/V plus normal action K/V by default,
|
| 8 |
+
matching the current training semantics.
|
| 9 |
+
|
| 10 |
+
Unsupported shapes route to the SDPA reference path. Native CUDA kernels are
|
| 11 |
+
not exposed until a shape-specialized implementation beats SDPA on the target
|
| 12 |
+
A100/5090 validation matrix.
|
| 13 |
+
"""
|
| 14 |
+
|
| 15 |
+
from __future__ import annotations
|
| 16 |
+
|
| 17 |
+
from typing import Optional
|
| 18 |
+
|
| 19 |
+
import torch
|
| 20 |
+
import torch.nn.functional as F
|
| 21 |
+
|
| 22 |
+
try:
|
| 23 |
+
from ._ops import ops
|
| 24 |
+
|
| 25 |
+
_HAS_OPS = hasattr(ops, "_flashrt_training_package_marker")
|
| 26 |
+
except Exception: # source-tree tests before kernel-builder creates _ops.py
|
| 27 |
+
ops = None
|
| 28 |
+
_HAS_OPS = False
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
MASK_VALUE_F32 = -2.3819763e38
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def _use_ops(namespace_ops) -> None:
|
| 35 |
+
"""Install a manually built extension (dev/testing path)."""
|
| 36 |
+
global ops, _HAS_OPS
|
| 37 |
+
ops = namespace_ops
|
| 38 |
+
_HAS_OPS = hasattr(ops, "_flashrt_training_package_marker")
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def _check_qkv(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor) -> None:
|
| 42 |
+
if q.dim() != 4 or k.dim() != 4 or v.dim() != 4:
|
| 43 |
+
raise ValueError("q, k, and v must be shaped (B, H, S, D)")
|
| 44 |
+
if q.shape[0] != k.shape[0] or q.shape[0] != v.shape[0]:
|
| 45 |
+
raise ValueError("q, k, and v batch dimensions must match")
|
| 46 |
+
if k.shape != v.shape:
|
| 47 |
+
raise ValueError("k and v shapes must match")
|
| 48 |
+
if q.shape[2] != k.shape[2] or q.shape[3] != k.shape[3]:
|
| 49 |
+
raise ValueError("q, k, and v sequence/head_dim dimensions must match")
|
| 50 |
+
if q.device != k.device or q.device != v.device:
|
| 51 |
+
raise ValueError("q, k, and v must be on the same device")
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def _as_valid(mask: Optional[torch.Tensor], batch: int, length: int, device: torch.device) -> torch.Tensor:
|
| 55 |
+
if mask is None:
|
| 56 |
+
return torch.ones((batch, length), dtype=torch.bool, device=device)
|
| 57 |
+
if mask.shape != (batch, length):
|
| 58 |
+
raise ValueError(f"mask must be shaped {(batch, length)}, got {tuple(mask.shape)}")
|
| 59 |
+
return mask.to(device=device, dtype=torch.bool)
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def build_block_sparse_bool_masks(
|
| 63 |
+
prefix_valid: Optional[torch.Tensor],
|
| 64 |
+
prefix_att: Optional[torch.Tensor],
|
| 65 |
+
*,
|
| 66 |
+
batch: int,
|
| 67 |
+
prefix_len: int,
|
| 68 |
+
action_len: int,
|
| 69 |
+
action_block_size: int,
|
| 70 |
+
non_fast_prefix_len: Optional[int] = None,
|
| 71 |
+
action_valid: Optional[torch.Tensor] = None,
|
| 72 |
+
device: Optional[torch.device] = None,
|
| 73 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 74 |
+
"""Build boolean masks for the split FlexAttention SDPA calls.
|
| 75 |
+
|
| 76 |
+
Returns ``(prefix_rows, action_rows)`` with shapes ``(B, P, S)`` and
|
| 77 |
+
``(B, A, S)``. Boolean True means the key/value position is visible.
|
| 78 |
+
|
| 79 |
+
``prefix_att`` follows Lerobot's cumulative-block convention: prefix key
|
| 80 |
+
``j`` is visible to prefix query ``i`` when ``cumsum(prefix_att)[j] <=
|
| 81 |
+
cumsum(prefix_att)[i]`` and both rows are valid. When omitted, prefix rows
|
| 82 |
+
attend to all valid prefix tokens.
|
| 83 |
+
"""
|
| 84 |
+
if action_block_size <= 0:
|
| 85 |
+
raise ValueError("action_block_size must be positive")
|
| 86 |
+
if prefix_len < 0 or action_len < 0:
|
| 87 |
+
raise ValueError("prefix_len and action_len must be non-negative")
|
| 88 |
+
total_len = prefix_len + action_len
|
| 89 |
+
dev = device
|
| 90 |
+
if dev is None:
|
| 91 |
+
for t in (prefix_valid, prefix_att, action_valid):
|
| 92 |
+
if t is not None:
|
| 93 |
+
dev = t.device
|
| 94 |
+
break
|
| 95 |
+
if dev is None:
|
| 96 |
+
dev = torch.device("cpu")
|
| 97 |
+
|
| 98 |
+
p_valid = _as_valid(prefix_valid, batch, prefix_len, dev)
|
| 99 |
+
a_valid = _as_valid(action_valid, batch, action_len, dev)
|
| 100 |
+
|
| 101 |
+
if prefix_att is None:
|
| 102 |
+
prefix_rows = p_valid[:, :, None] & p_valid[:, None, :]
|
| 103 |
+
else:
|
| 104 |
+
if prefix_att.shape != (batch, prefix_len):
|
| 105 |
+
raise ValueError(
|
| 106 |
+
f"prefix_att must be shaped {(batch, prefix_len)}, got {tuple(prefix_att.shape)}"
|
| 107 |
+
)
|
| 108 |
+
cum = torch.cumsum(prefix_att.to(device=dev, dtype=torch.long), dim=1)
|
| 109 |
+
prefix_rows = (cum[:, None, :] <= cum[:, :, None]) & p_valid[:, :, None] & p_valid[:, None, :]
|
| 110 |
+
|
| 111 |
+
prefix_pad = torch.zeros((batch, prefix_len, action_len), dtype=torch.bool, device=dev)
|
| 112 |
+
prefix_rows = torch.cat([prefix_rows, prefix_pad], dim=2)
|
| 113 |
+
|
| 114 |
+
nf = prefix_len if non_fast_prefix_len is None else int(non_fast_prefix_len)
|
| 115 |
+
nf = max(0, min(nf, prefix_len))
|
| 116 |
+
action_to_prefix = torch.zeros((batch, action_len, prefix_len), dtype=torch.bool, device=dev)
|
| 117 |
+
if nf > 0:
|
| 118 |
+
action_to_prefix[:, :, :nf] = p_valid[:, None, :nf]
|
| 119 |
+
action_to_prefix &= a_valid[:, :, None]
|
| 120 |
+
|
| 121 |
+
q_block = torch.arange(action_len, device=dev) // int(action_block_size)
|
| 122 |
+
kv_block = q_block
|
| 123 |
+
action_block = q_block[:, None] == kv_block[None, :]
|
| 124 |
+
action_block = action_block[None, :, :].expand(batch, -1, -1)
|
| 125 |
+
action_block = action_block & a_valid[:, :, None] & a_valid[:, None, :]
|
| 126 |
+
action_rows = torch.cat([action_to_prefix, action_block], dim=2)
|
| 127 |
+
|
| 128 |
+
if prefix_rows.shape != (batch, prefix_len, total_len):
|
| 129 |
+
raise AssertionError("internal prefix mask shape error")
|
| 130 |
+
if action_rows.shape != (batch, action_len, total_len):
|
| 131 |
+
raise AssertionError("internal action mask shape error")
|
| 132 |
+
return prefix_rows, action_rows
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
def _bool_to_sdpa_mask(mask: torch.Tensor, q: torch.Tensor) -> torch.Tensor:
|
| 136 |
+
value = MASK_VALUE_F32
|
| 137 |
+
if q.dtype.is_floating_point:
|
| 138 |
+
finfo = torch.finfo(q.dtype)
|
| 139 |
+
value = max(MASK_VALUE_F32, finfo.min)
|
| 140 |
+
return torch.where(
|
| 141 |
+
mask[:, None, :, :],
|
| 142 |
+
torch.zeros((), dtype=q.dtype, device=q.device),
|
| 143 |
+
torch.full((), value, dtype=q.dtype, device=q.device),
|
| 144 |
+
)
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
def _slice_attention_mask(
|
| 148 |
+
attention_mask: torch.Tensor,
|
| 149 |
+
start: int,
|
| 150 |
+
end: int,
|
| 151 |
+
q: torch.Tensor,
|
| 152 |
+
) -> torch.Tensor:
|
| 153 |
+
if attention_mask.dim() == 3:
|
| 154 |
+
mask = attention_mask[:, start:end, :]
|
| 155 |
+
if mask.dtype == torch.bool:
|
| 156 |
+
return mask[:, None, :, :]
|
| 157 |
+
return mask[:, None, :, :].to(dtype=q.dtype)
|
| 158 |
+
if attention_mask.dim() == 4:
|
| 159 |
+
mask = attention_mask[:, :, start:end, :]
|
| 160 |
+
return mask if mask.dtype == torch.bool else mask.to(dtype=q.dtype)
|
| 161 |
+
raise ValueError("attention_mask must be (B, S, S) or (B, 1|H, S, S)")
|
| 162 |
+
|
| 163 |
+
|
| 164 |
+
def _sdpa(
|
| 165 |
+
q: torch.Tensor,
|
| 166 |
+
k: torch.Tensor,
|
| 167 |
+
v: torch.Tensor,
|
| 168 |
+
mask: Optional[torch.Tensor],
|
| 169 |
+
*,
|
| 170 |
+
scale: Optional[float],
|
| 171 |
+
dropout_p: float,
|
| 172 |
+
enable_gqa: bool,
|
| 173 |
+
) -> torch.Tensor:
|
| 174 |
+
kwargs = {"attn_mask": mask, "dropout_p": float(dropout_p), "scale": scale}
|
| 175 |
+
if enable_gqa:
|
| 176 |
+
kwargs["enable_gqa"] = True
|
| 177 |
+
try:
|
| 178 |
+
return F.scaled_dot_product_attention(q, k, v, **kwargs)
|
| 179 |
+
except TypeError:
|
| 180 |
+
kwargs.pop("enable_gqa", None)
|
| 181 |
+
return F.scaled_dot_product_attention(q, k, v, **kwargs)
|
| 182 |
+
|
| 183 |
+
|
| 184 |
+
def reference_flex_attention(
|
| 185 |
+
q: torch.Tensor,
|
| 186 |
+
k: torch.Tensor,
|
| 187 |
+
v: torch.Tensor,
|
| 188 |
+
*,
|
| 189 |
+
prefix_len: int,
|
| 190 |
+
action_block_size: int,
|
| 191 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 192 |
+
prefix_valid: Optional[torch.Tensor] = None,
|
| 193 |
+
prefix_att: Optional[torch.Tensor] = None,
|
| 194 |
+
non_fast_prefix_len: Optional[int] = None,
|
| 195 |
+
action_valid: Optional[torch.Tensor] = None,
|
| 196 |
+
detach_prefix_kv_for_action: bool = True,
|
| 197 |
+
scale: Optional[float] = None,
|
| 198 |
+
dropout_p: float = 0.0,
|
| 199 |
+
enable_gqa: Optional[bool] = None,
|
| 200 |
+
) -> torch.Tensor:
|
| 201 |
+
"""SDPA reference for the PI052 FlexAttention replacement shape.
|
| 202 |
+
|
| 203 |
+
Args:
|
| 204 |
+
q, k, v: ``(B, Hq/Hkv, S, D)`` tensors.
|
| 205 |
+
prefix_len: number of prefix rows/columns at the start of sequence.
|
| 206 |
+
action_block_size: size of each block-diagonal action segment.
|
| 207 |
+
attention_mask: optional prebuilt additive or boolean full mask.
|
| 208 |
+
prefix_valid: optional ``(B, P)`` valid prefix positions.
|
| 209 |
+
prefix_att: optional ``(B, P)`` cumulative-block markers.
|
| 210 |
+
non_fast_prefix_len: prefix columns visible to action rows.
|
| 211 |
+
action_valid: optional ``(B, A)`` valid action positions.
|
| 212 |
+
detach_prefix_kv_for_action: detach prefix K/V on the action-row path.
|
| 213 |
+
scale: SDPA scale. Defaults to ``D ** -0.5``.
|
| 214 |
+
dropout_p: SDPA dropout probability.
|
| 215 |
+
enable_gqa: pass SDPA GQA mode when q heads and kv heads differ.
|
| 216 |
+
"""
|
| 217 |
+
_check_qkv(q, k, v)
|
| 218 |
+
batch, _, total_len, head_dim = q.shape
|
| 219 |
+
if not (0 <= int(prefix_len) <= total_len):
|
| 220 |
+
raise ValueError("prefix_len must be in [0, S]")
|
| 221 |
+
prefix_len = int(prefix_len)
|
| 222 |
+
action_len = total_len - prefix_len
|
| 223 |
+
if scale is None:
|
| 224 |
+
scale = head_dim**-0.5
|
| 225 |
+
if enable_gqa is None:
|
| 226 |
+
enable_gqa = q.shape[1] != k.shape[1]
|
| 227 |
+
|
| 228 |
+
q_prefix = q[:, :, :prefix_len, :]
|
| 229 |
+
q_action = q[:, :, prefix_len:, :]
|
| 230 |
+
k_prefix = k[:, :, :prefix_len, :]
|
| 231 |
+
k_action = k[:, :, prefix_len:, :]
|
| 232 |
+
v_prefix = v[:, :, :prefix_len, :]
|
| 233 |
+
v_action = v[:, :, prefix_len:, :]
|
| 234 |
+
|
| 235 |
+
if attention_mask is None:
|
| 236 |
+
prefix_bool, action_bool = build_block_sparse_bool_masks(
|
| 237 |
+
prefix_valid,
|
| 238 |
+
prefix_att,
|
| 239 |
+
batch=batch,
|
| 240 |
+
prefix_len=prefix_len,
|
| 241 |
+
action_len=action_len,
|
| 242 |
+
action_block_size=action_block_size,
|
| 243 |
+
non_fast_prefix_len=non_fast_prefix_len,
|
| 244 |
+
action_valid=action_valid,
|
| 245 |
+
device=q.device,
|
| 246 |
+
)
|
| 247 |
+
prefix_mask = _bool_to_sdpa_mask(prefix_bool, q)
|
| 248 |
+
action_mask = _bool_to_sdpa_mask(action_bool, q)
|
| 249 |
+
else:
|
| 250 |
+
prefix_mask = _slice_attention_mask(attention_mask, 0, prefix_len, q)
|
| 251 |
+
action_mask = _slice_attention_mask(attention_mask, prefix_len, total_len, q)
|
| 252 |
+
|
| 253 |
+
out_parts = []
|
| 254 |
+
if prefix_len:
|
| 255 |
+
out_parts.append(
|
| 256 |
+
_sdpa(
|
| 257 |
+
q_prefix,
|
| 258 |
+
k,
|
| 259 |
+
v,
|
| 260 |
+
prefix_mask,
|
| 261 |
+
scale=scale,
|
| 262 |
+
dropout_p=dropout_p,
|
| 263 |
+
enable_gqa=bool(enable_gqa),
|
| 264 |
+
)
|
| 265 |
+
)
|
| 266 |
+
if action_len:
|
| 267 |
+
prefix_k = k_prefix.detach() if detach_prefix_kv_for_action else k_prefix
|
| 268 |
+
prefix_v = v_prefix.detach() if detach_prefix_kv_for_action else v_prefix
|
| 269 |
+
k_for_action = torch.cat([prefix_k, k_action], dim=2)
|
| 270 |
+
v_for_action = torch.cat([prefix_v, v_action], dim=2)
|
| 271 |
+
out_parts.append(
|
| 272 |
+
_sdpa(
|
| 273 |
+
q_action,
|
| 274 |
+
k_for_action,
|
| 275 |
+
v_for_action,
|
| 276 |
+
action_mask,
|
| 277 |
+
scale=scale,
|
| 278 |
+
dropout_p=dropout_p,
|
| 279 |
+
enable_gqa=bool(enable_gqa),
|
| 280 |
+
)
|
| 281 |
+
)
|
| 282 |
+
if not out_parts:
|
| 283 |
+
return q.new_empty(q.shape)
|
| 284 |
+
return torch.cat(out_parts, dim=2) if len(out_parts) == 2 else out_parts[0]
|
| 285 |
+
|
| 286 |
+
|
| 287 |
+
def _manual_attention_part(qs, ks, vs, mask, scale):
|
| 288 |
+
"""Materialized-logits attention part: cuBLAS GEMMs + fused masked softmax.
|
| 289 |
+
|
| 290 |
+
Same math as SDPA with an additive mask (fp32 softmax; logits stored in
|
| 291 |
+
the io dtype between the GEMM and the softmax). Grouped queries run as a
|
| 292 |
+
strided batched GEMM over the KV heads, so a 1-head K/V is never
|
| 293 |
+
repeated. At PI052 training shapes (GQA 8:1, D=256, bf16) this beats
|
| 294 |
+
both SDPA-with-dense-mask (2.3-3.1x) and the best FlexAttention
|
| 295 |
+
configuration (1.4-2.9x) on fwd+bwd — see benchmarks/RESULTS.md.
|
| 296 |
+
"""
|
| 297 |
+
B, H, Sq, D = qs.shape
|
| 298 |
+
Hk = ks.shape[1]
|
| 299 |
+
if Hk != H:
|
| 300 |
+
g = H // Hk
|
| 301 |
+
q2 = qs.reshape(B, Hk, g * Sq, D)
|
| 302 |
+
logits = (q2 @ ks.transpose(-1, -2)).reshape(B, H, Sq, -1)
|
| 303 |
+
else:
|
| 304 |
+
logits = qs @ ks.transpose(-1, -2)
|
| 305 |
+
logits = logits * scale
|
| 306 |
+
if mask is not None:
|
| 307 |
+
logits = logits + mask
|
| 308 |
+
p = logits.float().softmax(dim=-1).to(qs.dtype)
|
| 309 |
+
if Hk != H:
|
| 310 |
+
out = (p.reshape(B, Hk, g * Sq, -1) @ vs).reshape(B, H, Sq, D)
|
| 311 |
+
else:
|
| 312 |
+
out = p @ vs
|
| 313 |
+
return out
|
| 314 |
+
|
| 315 |
+
|
| 316 |
+
_manual_part_compiled = None
|
| 317 |
+
|
| 318 |
+
|
| 319 |
+
def _get_manual_part():
|
| 320 |
+
global _manual_part_compiled
|
| 321 |
+
if _manual_part_compiled is None:
|
| 322 |
+
_manual_part_compiled = torch.compile(_manual_attention_part, dynamic=False)
|
| 323 |
+
return _manual_part_compiled
|
| 324 |
+
|
| 325 |
+
|
| 326 |
+
def manual_attention(
|
| 327 |
+
q: torch.Tensor,
|
| 328 |
+
k: torch.Tensor,
|
| 329 |
+
v: torch.Tensor,
|
| 330 |
+
*,
|
| 331 |
+
prefix_len: int,
|
| 332 |
+
action_block_size: int,
|
| 333 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 334 |
+
prefix_valid: Optional[torch.Tensor] = None,
|
| 335 |
+
prefix_att: Optional[torch.Tensor] = None,
|
| 336 |
+
non_fast_prefix_len: Optional[int] = None,
|
| 337 |
+
action_valid: Optional[torch.Tensor] = None,
|
| 338 |
+
detach_prefix_kv_for_action: bool = True,
|
| 339 |
+
scale: Optional[float] = None,
|
| 340 |
+
dropout_p: float = 0.0,
|
| 341 |
+
compile_part: bool = True,
|
| 342 |
+
) -> torch.Tensor:
|
| 343 |
+
"""Materialized-logits implementation of :func:`reference_flex_attention`.
|
| 344 |
+
|
| 345 |
+
Same mask semantics and prefix/action split; each part runs through
|
| 346 |
+
:func:`_manual_attention_part` instead of SDPA. ``dropout_p`` must be 0
|
| 347 |
+
(training attention dropout is unused in PI052); other values raise so
|
| 348 |
+
callers fall back explicitly.
|
| 349 |
+
"""
|
| 350 |
+
if dropout_p:
|
| 351 |
+
raise ValueError("manual_attention does not support dropout; use the reference path")
|
| 352 |
+
_check_qkv(q, k, v)
|
| 353 |
+
batch, _, total_len, head_dim = q.shape
|
| 354 |
+
if not (0 <= int(prefix_len) <= total_len):
|
| 355 |
+
raise ValueError("prefix_len must be in [0, S]")
|
| 356 |
+
prefix_len = int(prefix_len)
|
| 357 |
+
action_len = total_len - prefix_len
|
| 358 |
+
if scale is None:
|
| 359 |
+
scale = head_dim**-0.5
|
| 360 |
+
|
| 361 |
+
if attention_mask is None:
|
| 362 |
+
prefix_bool, action_bool = build_block_sparse_bool_masks(
|
| 363 |
+
prefix_valid,
|
| 364 |
+
prefix_att,
|
| 365 |
+
batch=batch,
|
| 366 |
+
prefix_len=prefix_len,
|
| 367 |
+
action_len=action_len,
|
| 368 |
+
action_block_size=action_block_size,
|
| 369 |
+
non_fast_prefix_len=non_fast_prefix_len,
|
| 370 |
+
action_valid=action_valid,
|
| 371 |
+
device=q.device,
|
| 372 |
+
)
|
| 373 |
+
prefix_mask = _bool_to_sdpa_mask(prefix_bool, q)
|
| 374 |
+
action_mask = _bool_to_sdpa_mask(action_bool, q)
|
| 375 |
+
else:
|
| 376 |
+
prefix_mask = _slice_attention_mask(attention_mask, 0, prefix_len, q)
|
| 377 |
+
action_mask = _slice_attention_mask(attention_mask, prefix_len, total_len, q)
|
| 378 |
+
if prefix_mask.dtype == torch.bool:
|
| 379 |
+
prefix_mask = _bool_to_sdpa_mask(prefix_mask[:, 0], q)
|
| 380 |
+
if action_mask.dtype == torch.bool:
|
| 381 |
+
action_mask = _bool_to_sdpa_mask(action_mask[:, 0], q)
|
| 382 |
+
|
| 383 |
+
part = _get_manual_part() if compile_part else _manual_attention_part
|
| 384 |
+
out_parts = []
|
| 385 |
+
if prefix_len:
|
| 386 |
+
out_parts.append(part(q[:, :, :prefix_len, :], k, v, prefix_mask, scale))
|
| 387 |
+
if action_len:
|
| 388 |
+
k_prefix = k[:, :, :prefix_len, :]
|
| 389 |
+
v_prefix = v[:, :, :prefix_len, :]
|
| 390 |
+
if detach_prefix_kv_for_action:
|
| 391 |
+
k_prefix = k_prefix.detach()
|
| 392 |
+
v_prefix = v_prefix.detach()
|
| 393 |
+
k_for_action = torch.cat([k_prefix, k[:, :, prefix_len:, :]], dim=2)
|
| 394 |
+
v_for_action = torch.cat([v_prefix, v[:, :, prefix_len:, :]], dim=2)
|
| 395 |
+
out_parts.append(part(q[:, :, prefix_len:, :], k_for_action, v_for_action, action_mask, scale))
|
| 396 |
+
if not out_parts:
|
| 397 |
+
return q.new_empty(q.shape)
|
| 398 |
+
return torch.cat(out_parts, dim=2) if len(out_parts) == 2 else out_parts[0]
|
| 399 |
+
|
| 400 |
+
|
| 401 |
+
def flex_attention(
|
| 402 |
+
q: torch.Tensor,
|
| 403 |
+
k: torch.Tensor,
|
| 404 |
+
v: torch.Tensor,
|
| 405 |
+
*,
|
| 406 |
+
prefix_len: int,
|
| 407 |
+
action_block_size: int,
|
| 408 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 409 |
+
prefix_valid: Optional[torch.Tensor] = None,
|
| 410 |
+
prefix_att: Optional[torch.Tensor] = None,
|
| 411 |
+
non_fast_prefix_len: Optional[int] = None,
|
| 412 |
+
action_valid: Optional[torch.Tensor] = None,
|
| 413 |
+
detach_prefix_kv_for_action: bool = True,
|
| 414 |
+
scale: Optional[float] = None,
|
| 415 |
+
dropout_p: float = 0.0,
|
| 416 |
+
enable_gqa: Optional[bool] = None,
|
| 417 |
+
force_fallback: bool = False,
|
| 418 |
+
impl: str = "sdpa",
|
| 419 |
+
) -> torch.Tensor:
|
| 420 |
+
"""Flex-style block-sparse attention.
|
| 421 |
+
|
| 422 |
+
``impl="sdpa"`` (default) keeps the SDPA reference path;
|
| 423 |
+
``impl="manual"`` routes through the materialized-logits
|
| 424 |
+
implementation; ``impl="auto"`` picks manual only where it has been
|
| 425 |
+
measured to win end-to-end — consumer Blackwell (sm120-class) with
|
| 426 |
+
no dropout. On A100 (sm80) the manual math wins microbenches but
|
| 427 |
+
loses training-step integration, and on H100/H200 (sm90) the fused
|
| 428 |
+
FMHA kernels win outright, so auto keeps SDPA there.
|
| 429 |
+
"""
|
| 430 |
+
_ = force_fallback
|
| 431 |
+
if impl == "auto":
|
| 432 |
+
sm120 = q.is_cuda and torch.cuda.get_device_capability(q.device)[0] == 12
|
| 433 |
+
impl = "manual" if (sm120 and not dropout_p) else "sdpa"
|
| 434 |
+
if impl == "manual":
|
| 435 |
+
return manual_attention(
|
| 436 |
+
q,
|
| 437 |
+
k,
|
| 438 |
+
v,
|
| 439 |
+
prefix_len=prefix_len,
|
| 440 |
+
action_block_size=action_block_size,
|
| 441 |
+
attention_mask=attention_mask,
|
| 442 |
+
prefix_valid=prefix_valid,
|
| 443 |
+
prefix_att=prefix_att,
|
| 444 |
+
non_fast_prefix_len=non_fast_prefix_len,
|
| 445 |
+
action_valid=action_valid,
|
| 446 |
+
detach_prefix_kv_for_action=detach_prefix_kv_for_action,
|
| 447 |
+
scale=scale,
|
| 448 |
+
dropout_p=dropout_p,
|
| 449 |
+
)
|
| 450 |
+
return reference_flex_attention(
|
| 451 |
+
q,
|
| 452 |
+
k,
|
| 453 |
+
v,
|
| 454 |
+
prefix_len=prefix_len,
|
| 455 |
+
action_block_size=action_block_size,
|
| 456 |
+
attention_mask=attention_mask,
|
| 457 |
+
prefix_valid=prefix_valid,
|
| 458 |
+
prefix_att=prefix_att,
|
| 459 |
+
non_fast_prefix_len=non_fast_prefix_len,
|
| 460 |
+
action_valid=action_valid,
|
| 461 |
+
detach_prefix_kv_for_action=detach_prefix_kv_for_action,
|
| 462 |
+
scale=scale,
|
| 463 |
+
dropout_p=dropout_p,
|
| 464 |
+
enable_gqa=enable_gqa,
|
| 465 |
+
)
|
| 466 |
+
|
| 467 |
+
|
| 468 |
+
def flex_attention_forward(*args, **kwargs) -> torch.Tensor:
|
| 469 |
+
"""Forward-only compatibility wrapper."""
|
| 470 |
+
return flex_attention(*args, **kwargs)
|
| 471 |
+
|
| 472 |
+
|
| 473 |
+
def backend_marker(x: torch.Tensor) -> torch.Tensor:
|
| 474 |
+
if ops is None:
|
| 475 |
+
return x
|
| 476 |
+
return ops._flashrt_training_package_marker(x)
|
| 477 |
+
|
| 478 |
+
|
| 479 |
+
__all__ = [
|
| 480 |
+
"MASK_VALUE_F32",
|
| 481 |
+
"backend_marker",
|
| 482 |
+
"build_block_sparse_bool_masks",
|
| 483 |
+
"flex_attention",
|
| 484 |
+
"flex_attention_forward",
|
| 485 |
+
"manual_attention",
|
| 486 |
+
"reference_flex_attention",
|
| 487 |
+
]
|
build/torch212-cxx11-cu132-x86_64-linux/_flashrt_flex_attention_train_cuda_eaf7454.abi3.so
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:57dd5d0475cf4953b7f770824ce720676641df6cd59ee9b95aef3564a8fb2435
|
| 3 |
+
size 56800
|
build/torch212-cxx11-cu132-x86_64-linux/_ops.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
from . import _flashrt_flex_attention_train_cuda_eaf7454
|
| 3 |
+
ops = torch.ops._flashrt_flex_attention_train_cuda_eaf7454
|
| 4 |
+
|
| 5 |
+
def add_op_namespace_prefix(op_name: str):
|
| 6 |
+
"""
|
| 7 |
+
Prefix op by namespace.
|
| 8 |
+
"""
|
| 9 |
+
return f"_flashrt_flex_attention_train_cuda_eaf7454::{op_name}"
|
build/torch212-cxx11-cu132-x86_64-linux/flashrt_flex_attention_train/__init__.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import ctypes
|
| 2 |
+
import importlib.util
|
| 3 |
+
import sys
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
from types import ModuleType
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def _import_from_path(file_path: Path) -> ModuleType:
|
| 9 |
+
# We cannot use the module name as-is, after adding it to `sys.modules`,
|
| 10 |
+
# it would also be used for other imports. So, we make a module name that
|
| 11 |
+
# depends on the path for it to be unique using the hex-encoded hash of
|
| 12 |
+
# the path.
|
| 13 |
+
path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
|
| 14 |
+
module_name = path_hash
|
| 15 |
+
spec = importlib.util.spec_from_file_location(module_name, file_path)
|
| 16 |
+
if spec is None:
|
| 17 |
+
raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
|
| 18 |
+
module = importlib.util.module_from_spec(spec)
|
| 19 |
+
if module is None:
|
| 20 |
+
raise ImportError(f"Cannot load module {module_name} from spec")
|
| 21 |
+
sys.modules[module_name] = module
|
| 22 |
+
spec.loader.exec_module(module) # type: ignore
|
| 23 |
+
return module
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
|
build/torch212-cxx11-cu132-x86_64-linux/metadata.json
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "flashrt-flex-attention-train",
|
| 3 |
+
"id": "_flashrt_flex_attention_train_cuda_eaf7454",
|
| 4 |
+
"version": 1,
|
| 5 |
+
"license": "Apache-2.0",
|
| 6 |
+
"python-depends": [],
|
| 7 |
+
"backend": {
|
| 8 |
+
"type": "cuda",
|
| 9 |
+
"archs": [
|
| 10 |
+
"10.0",
|
| 11 |
+
"11.0",
|
| 12 |
+
"12.0",
|
| 13 |
+
"12.1+PTX",
|
| 14 |
+
"7.5",
|
| 15 |
+
"8.0",
|
| 16 |
+
"8.6",
|
| 17 |
+
"8.7",
|
| 18 |
+
"8.9",
|
| 19 |
+
"9.0"
|
| 20 |
+
]
|
| 21 |
+
}
|
| 22 |
+
}
|