Kernels:
Trusted publisher
Uploaded using `kernel-builder`.
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- build/torch210-rocm70-x86_64-linux/__init__.py +27 -0
- build/torch210-rocm70-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so +3 -0
- build/torch210-rocm70-x86_64-linux/_ops.py +9 -0
- build/torch210-rocm70-x86_64-linux/metadata.json +37 -0
- build/torch210-rocm70-x86_64-linux/metadata.json.sigstore +1 -0
- build/torch210-rocm70-x86_64-linux/mha.py +332 -0
- build/torch210-rocm70-x86_64-linux/symbols.json +349 -0
- build/torch210-rocm71-x86_64-linux/__init__.py +27 -0
- build/torch210-rocm71-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so +3 -0
- build/torch210-rocm71-x86_64-linux/_ops.py +9 -0
- build/torch210-rocm71-x86_64-linux/metadata.json +37 -0
- build/torch210-rocm71-x86_64-linux/metadata.json.sigstore +1 -0
- build/torch210-rocm71-x86_64-linux/mha.py +332 -0
- build/torch210-rocm71-x86_64-linux/symbols.json +349 -0
- build/torch211-rocm71-x86_64-linux/__init__.py +27 -0
- build/torch211-rocm71-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so +3 -0
- build/torch211-rocm71-x86_64-linux/_ops.py +9 -0
- build/torch211-rocm71-x86_64-linux/metadata.json +37 -0
- build/torch211-rocm71-x86_64-linux/metadata.json.sigstore +1 -0
- build/torch211-rocm71-x86_64-linux/mha.py +332 -0
- build/torch211-rocm71-x86_64-linux/symbols.json +349 -0
- build/torch211-rocm72-x86_64-linux/__init__.py +27 -0
- build/torch211-rocm72-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so +3 -0
- build/torch211-rocm72-x86_64-linux/_ops.py +9 -0
- build/torch211-rocm72-x86_64-linux/metadata.json +37 -0
- build/torch211-rocm72-x86_64-linux/metadata.json.sigstore +1 -0
- build/torch211-rocm72-x86_64-linux/mha.py +332 -0
- build/torch211-rocm72-x86_64-linux/symbols.json +349 -0
- build/torch212-rocm71-x86_64-linux/__init__.py +27 -0
- build/torch212-rocm71-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so +3 -0
- build/torch212-rocm71-x86_64-linux/_ops.py +9 -0
- build/torch212-rocm71-x86_64-linux/metadata.json +37 -0
- build/torch212-rocm71-x86_64-linux/metadata.json.sigstore +1 -0
- build/torch212-rocm71-x86_64-linux/mha.py +332 -0
- build/torch212-rocm71-x86_64-linux/symbols.json +349 -0
- build/torch212-rocm72-x86_64-linux/__init__.py +27 -0
- build/torch212-rocm72-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so +3 -0
- build/torch212-rocm72-x86_64-linux/_ops.py +9 -0
- build/torch212-rocm72-x86_64-linux/metadata.json +37 -0
- build/torch212-rocm72-x86_64-linux/metadata.json.sigstore +1 -0
- build/torch212-rocm72-x86_64-linux/mha.py +332 -0
- build/torch212-rocm72-x86_64-linux/symbols.json +349 -0
- build/torch213-rocm71-x86_64-linux/__init__.py +27 -0
- build/torch213-rocm71-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so +3 -0
- build/torch213-rocm71-x86_64-linux/_ops.py +9 -0
- build/torch213-rocm71-x86_64-linux/metadata.json +37 -0
- build/torch213-rocm71-x86_64-linux/metadata.json.sigstore +1 -0
- build/torch213-rocm71-x86_64-linux/mha.py +332 -0
- build/torch213-rocm71-x86_64-linux/symbols.json +349 -0
- build/torch213-rocm72-x86_64-linux/__init__.py +27 -0
build/torch210-rocm70-x86_64-linux/__init__.py
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""AITER Composable-Kernel FlashAttention kernels for AMD ROCm (compiled HIP).
|
| 2 |
+
|
| 3 |
+
Compiled (Composable Kernel / HIP) counterpart to the Triton-based
|
| 4 |
+
``aiter-flash-attn`` package, repackaged from the CK FMHA forward path of the
|
| 5 |
+
`ROCm/aiter <https://github.com/ROCm/aiter>`_ project as a self-contained
|
| 6 |
+
Hugging Face Hub kernel. Exposes the FlashAttention forward entry points with
|
| 7 |
+
learnable attention sink support (e.g. gpt-oss).
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
from .mha import (
|
| 11 |
+
flash_attn_func,
|
| 12 |
+
flash_attn_varlen_func,
|
| 13 |
+
flash_attn_with_kvcache,
|
| 14 |
+
)
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
__kernel_metadata__ = {
|
| 18 |
+
"license": "mit",
|
| 19 |
+
}
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
__all__ = [
|
| 23 |
+
"__kernel_metadata__",
|
| 24 |
+
"flash_attn_func",
|
| 25 |
+
"flash_attn_varlen_func",
|
| 26 |
+
"flash_attn_with_kvcache",
|
| 27 |
+
]
|
build/torch210-rocm70-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:8edc92287fd49261db7c4799b29691a77e7db509ec44dac8bf85fba127f0e641
|
| 3 |
+
size 204329576
|
build/torch210-rocm70-x86_64-linux/_ops.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
from . import _aiter_flash_attn_ck_rocm_fe2bb66
|
| 3 |
+
ops = torch.ops._aiter_flash_attn_ck_rocm_fe2bb66
|
| 4 |
+
|
| 5 |
+
def add_op_namespace_prefix(op_name: str):
|
| 6 |
+
"""
|
| 7 |
+
Prefix op by namespace.
|
| 8 |
+
"""
|
| 9 |
+
return f"_aiter_flash_attn_ck_rocm_fe2bb66::{op_name}"
|
build/torch210-rocm70-x86_64-linux/metadata.json
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "aiter-flash-attn-ck",
|
| 3 |
+
"id": "_aiter_flash_attn_ck_rocm_fe2bb66",
|
| 4 |
+
"version": 1,
|
| 5 |
+
"kernels-minver": "0.14.0",
|
| 6 |
+
"license": "MIT",
|
| 7 |
+
"python-depends": [],
|
| 8 |
+
"kernel-depends": [],
|
| 9 |
+
"backend": {
|
| 10 |
+
"type": "rocm",
|
| 11 |
+
"archs": [
|
| 12 |
+
"gfx942",
|
| 13 |
+
"gfx950"
|
| 14 |
+
]
|
| 15 |
+
},
|
| 16 |
+
"digest": {
|
| 17 |
+
"algorithm": "sha256",
|
| 18 |
+
"files": {
|
| 19 |
+
"__init__.py": "7wj1GTm2bqTLLCyT4xLuAk/+JWYrhINjY/ODMNWhgZ0=",
|
| 20 |
+
"_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so": "jtySKH/UkmHbfEeZspaRp359tQnsRNrIv4X7oSfw5kE=",
|
| 21 |
+
"_ops.py": "4WkdYDWWJp60zep+oW/5ym9vIeVMNfrgRCkP3o/w7yU=",
|
| 22 |
+
"mha.py": "zCAjY4O+DiCzmEL+W9sqXxZaqjsj4RJbWFcyPpUZejk=",
|
| 23 |
+
"symbols.json": "rhnF4H8Jsb6I+YPxIVFMgnLcTQJODEXbZ7ge77NZXAU="
|
| 24 |
+
}
|
| 25 |
+
},
|
| 26 |
+
"provenance": {
|
| 27 |
+
"kernel-builder": {
|
| 28 |
+
"version": "0.18.0-dev0",
|
| 29 |
+
"commit": "d054906a13710b811b96697b4635f03020842733",
|
| 30 |
+
"dirty": false
|
| 31 |
+
},
|
| 32 |
+
"kernel": {
|
| 33 |
+
"commit": "fe2bb66fee6c2a77eccf08a0508dc57428d488ed",
|
| 34 |
+
"dirty": false
|
| 35 |
+
}
|
| 36 |
+
}
|
| 37 |
+
}
|
build/torch210-rocm70-x86_64-linux/metadata.json.sigstore
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json", "verificationMaterial":{"certificate":{"rawBytes":"MIIHLDCCBrOgAwIBAgIURaYLBvCxo8VXbDXCmafFy5NEz4IwCgYIKoZIzj0EAwMwNzEVMBMGA1UEChMMc2lnc3RvcmUuZGV2MR4wHAYDVQQDExVzaWdzdG9yZS1pbnRlcm1lZGlhdGUwHhcNMjYxMDE5MjIzMTEwWhcNMjYxMDE5MjI0MTEwWjAAMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEj/NhfZLBdSTQaFcS2gIM0IgWG5P9kCv+yNljBxYjIPdh5dPlT5SG3L0TOIhLASBQ2EJ2qqDHsaMED9/WJmH9hqOCBdIwggXOMA4GA1UdDwEB/wQEAwIHgDATBgNVHSUEDDAKBggrBgEFBQcDAzAfBgNVHSMEGDAWgBTf0+nPViQRlvmo2OkoVaLGLhhkPzBrBgNVHREBAf8EYTBfhl1odHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkvLmdpdGh1Yi93b3JrZmxvd3MvYnVpbGQueWFtbEByZWZzL2hlYWRzL21haW4wOQYKKwYBBAGDvzABAQQraHR0cHM6Ly90b2tlbi5hY3Rpb25zLmdpdGh1YnVzZXJjb250ZW50LmNvbTAfBgorBgEEAYO/MAECBBF3b3JrZmxvd19kaXNwYXRjaDA2BgorBgEEAYO/MAEDBChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMBMGCisGAQQBg78wAQQEBUJ1aWxkMCsGCisGAQQBg78wAQUEHWh1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5MB0GCisGAQQBg78wAQYED3JlZnMvaGVhZHMvbWFpbjA7BgorBgEEAYO/MAEIBC0MK2h0dHBzOi8vdG9rZW4uYWN0aW9ucy5naXRodWJ1c2VyY29udGVudC5jb20wbQYKKwYBBAGDvzABCQRfDF1odHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkvLmdpdGh1Yi93b3JrZmxvd3MvYnVpbGQueWFtbEByZWZzL2hlYWRzL21haW4wOAYKKwYBBAGDvzABCgQqDChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMBsGCisGAQQBg78wAQsEDQwLc2VsZi1ob3N0ZWQwQAYKKwYBBAGDvzABDAQyDDBodHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkwOAYKKwYBBAGDvzABDQQqDChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMB8GCisGAQQBg78wAQ4EEQwPcmVmcy9oZWFkcy9tYWluMBoGCisGAQQBg78wAQ8EDAwKMTA3MTQ3NTUyOTAuBgorBgEEAYO/MAEQBCAMHmh0dHBzOi8vZ2l0aHViLmNvbS9odWdnaW5nZmFjZTAYBgorBgEEAYO/MAERBAoMCDI1NzIwNzQzMG0GCisGAQQBg78wARIEXwxdaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5Ly5naXRodWIvd29ya2Zsb3dzL2J1aWxkLnlhbWxAcmVmcy9oZWFkcy9tYWluMDgGCisGAQQBg78wARMEKgwoZmUyYmI2NmZlZTZjMmE3N2VjY2YwOGEwNTA4ZGM1NzQyOGQ0ODhlZDAhBgorBgEEAYO/MAEUBBMMEXdvcmtmbG93X2Rpc3BhdGNoMGQGCisGAQQBg78wARUEVgxUaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5L2FjdGlvbnMvcnVucy8zOTIzOTUxODEwNC9hdHRlbXB0cy8xMBYGCisGAQQBg78wARYECAwGcHVibGljMEYGCisGAQQBg78wARgEOAw2cmVwbzpodWdnaW5nZmFjZS9rZXJuZWxzLWNvbW11bml0eTpyZWY6cmVmcy9oZWFkcy9tYWluMIGLBgorBgEEAdZ5AgQCBH0EewB5AHcA3T0wasbHETJjGR4cmWc3AqJKXrjePK3/h4pygC8p7o4AAAGhVkpFsgAABAMASDBGAiEAs9cSP0P/pRZ4prabNNIq89TS0RhX2f3JRpHryeIhR3YCIQDnQuuXxSnnnfl8Q7URBS7DC23YrDaSpshSRUENXthSezAKBggqhkjOPQQDAwNnADBkAjEA4gCCe/00DKR/78FksRs11BgMWSSsisk8ME6OMyx2ZOKIvMVy1VHI15E6Xfr3isA+Ai9ZI6Otv6AnOVw/zUHKLvuj6VoYdLp26TwhWYN2N8LFZjzVGNX+0NcN8O6rYcPUsw=="}, "tlogEntries":[{"logIndex":"3327211074", "logId":{"keyId":"wNI9atQGlz+VWfO6LRygH4QUfY/8W4RFwiT5i5WRgB0="}, "kindVersion":{"kind":"hashedrekord", "version":"0.0.1"}, "integratedTime":"1792449070", "inclusionPromise":{"signedEntryTimestamp":"MEUCIBvKLoqT8cbWQvPal/N1/qcIkVmZMOCOjE6UEkoLG/MqAiEAviJ70KCt6sreW85HkoeGGZY7csZL/nOGK/uLJB3Mo5Y="}, "inclusionProof":{"logIndex":"3205306812", "rootHash":"95EwDSNtaCqCTTYN6UQnUbg7LfPk37Dr84DcE4k1RBM=", "treeSize":"3205306841", "hashes":["Ocn28evxoK5pHLcp+On0tDY2wedQVBaFskplXSixJIA=", "mwzUJi5lok6Hybu0CxcVdWew/1Qbhdd1iGjTjEYBcZY=", "5H4knr1LMewHY/JuNzQYKyCx9Q74EBBYz3NqEsl4bQo=", "QZRIvZ2WmlY+tW/q10eYuY+Pr8vI5My3TuMAw/2EObw=", "k+ztsFzU6f6Y66AdU9NwlAXHqLCPJ6LKyl669zO1Hr4=", "ozBdZyx9tTLcaQf2DOlQzhq1u90wECTuyt20Ufxet/w=", "ddB/VAJce0tlDaSu+1n3gD4CW0cuWQCIYTwHV3ehNRg=", "mnhxYPYaPxA2/IDhvMQ+URxfzi+3aVt0cMthNxkCByA=", "jhDPXc61HrM2wJE+UCBVCldN7S5Fn1BuEfb57UT4fLc=", "5egUlZ4Pi0Ix/qLyCDXocsWywJCHqrde/S06vOZvlOM=", "QsXufUpoKtSjKAB19K1G00umQpWvPHlBNuRGFiiVjQQ=", "QIMkwQbwAewhLJd//5FFwI3SnyNQBGjCgJrelfG8vhc=", "Et17CACUv9QN64irUJuBjsrgRGkGnnJ9xacRV/aBdwQ=", "lqgEwkxhDMs/gdIAE4lmjfMwg7nDmv7lda6O0LGliCw=", "HovTPBWzoErvUg1u5sIi3x1ZonZ7yJekVsBrmpomCIA=", "G+nk7mgE5h6dUmv2UY7NDJrY3WvrZY31raaO1AETt+o=", "qlTWNDzERnTBQVcHHXXVPjiUP0SsikGxfcxVlgqVFDo=", "gg9GNApNke3+zNhl4tsWtmIycCFrOMCB3lys6XuSQNQ=", "NlOclldmKPA1tMKIskqHAwuQDYHp31YwpK+OOMrMccc=", "qxzHanAzz57SDdmJe0B7bJK72NTIBbwEMGBBKvDOROw=", "xH/DCseLHr9eKoYT8qsORZK7zVdEGYWHuVtsVrD95wY="], "checkpoint":{"envelope":"rekor.sigstore.dev - 1193050959916656506\n3205306841\n95EwDSNtaCqCTTYN6UQnUbg7LfPk37Dr84DcE4k1RBM=\n\n— rekor.sigstore.dev wNI9ajBFAiEAr3pHRHsXHEeGuhek+3BjAqkphYe6M+vo6+TlCdjCZNYCIE11bmNslf4u1xUYgIQHRJqsJ7XyHOBWS7628PjJh16z\n"}}, "canonicalizedBody":"eyJhcGlWZXJzaW9uIjoiMC4wLjEiLCJraW5kIjoiaGFzaGVkcmVrb3JkIiwic3BlYyI6eyJkYXRhIjp7Imhhc2giOnsiYWxnb3JpdGhtIjoic2hhMjU2IiwidmFsdWUiOiIwOGZhMGNkNTBhODRmOTQxMjc2YzVlNmQ1ZmE0OWYxZDRlMDFlMTU5ODdhNjBhYWI4ZjJhYjZlMzE5ZWVjNzdiIn19LCJzaWduYXR1cmUiOnsiY29udGVudCI6Ik1FVUNJQ1BTaWFYOXBZL29aYnp1YXNHQW5PUTB1TDhQL0pMSEsrRkttVUhZZnZUMkFpRUFzOEZDOEhSeVZFZmo3QUdqYnNJNlZmb2FoY2NLWHM0RkNmY3luZ1RzUVhjPSIsInB1YmxpY0tleSI6eyJjb250ZW50IjoiTFMwdExTMUNSVWRKVGlCRFJWSlVTVVpKUTBGVVJTMHRMUzB0Q2sxSlNVaE1SRU5EUW5KUFowRjNTVUpCWjBsVlVtRlpURUoyUTNodk9GWllZa1JZUTIxaFprWjVOVTVGZWpSSmQwTm5XVWxMYjFwSmVtb3dSVUYzVFhjS1RucEZWazFDVFVkQk1WVkZRMmhOVFdNeWJHNWpNMUoyWTIxVmRWcEhWakpOVWpSM1NFRlpSRlpSVVVSRmVGWjZZVmRrZW1SSE9YbGFVekZ3WW01U2JBcGpiVEZzV2tkc2FHUkhWWGRJYUdOT1RXcFplRTFFUlRWTmFrbDZUVlJGZDFkb1kwNU5hbGw0VFVSRk5VMXFTVEJOVkVWM1YycEJRVTFHYTNkRmQxbElDa3R2V2tsNmFqQkRRVkZaU1V0dldrbDZhakJFUVZGalJGRm5RVVZxTDA1b1pscE1RbVJUVkZGaFJtTlRNbWRKVFRCSloxZEhOVkE1YTBOMkszbE9iR29LUW5oWmFrbFFaR2cxWkZCc1ZEVlRSek5NTUZSUFNXaE1RVk5DVVRKRlNqSnhjVVJJYzJGTlJVUTVMMWRLYlVnNWFIRlBRMEprU1hkbloxaFBUVUUwUndwQk1WVmtSSGRGUWk5M1VVVkJkMGxJWjBSQlZFSm5UbFpJVTFWRlJFUkJTMEpuWjNKQ1owVkdRbEZqUkVGNlFXWkNaMDVXU0ZOTlJVZEVRVmRuUWxSbUNqQXJibEJXYVZGU2JIWnRiekpQYTI5V1lVeEhUR2hvYTFCNlFuSkNaMDVXU0ZKRlFrRm1PRVZaVkVKbWFHd3hiMlJJVW5kamVtOTJUREprY0dSSGFERUtXV2sxYW1JeU1IWmhTRlp1V2pKc2RWb3lXbWhaTWxWMllUSldlV0p0Vm5OamVURnFZakl4ZEdSWE5YQmtTR3QyVEcxa2NHUkhhREZaYVRrellqTktjZ3BhYlhoMlpETk5kbGx1Vm5CaVIxRjFaVmRHZEdKRlFubGFWMXA2VERKb2JGbFhVbnBNTWpGb1lWYzBkMDlSV1V0TGQxbENRa0ZIUkhaNlFVSkJVVkZ5Q21GSVVqQmpTRTAyVEhrNU1HSXlkR3hpYVRWb1dUTlNjR0l5TlhwTWJXUndaRWRvTVZsdVZucGFXRXBxWWpJMU1GcFhOVEJNYlU1MllsUkJaa0puYjNJS1FtZEZSVUZaVHk5TlFVVkRRa0pHTTJJelNuSmFiWGgyWkRFNWEyRllUbmRaV0ZKcVlVUkJNa0puYjNKQ1owVkZRVmxQTDAxQlJVUkNRMmh0V2xSS2FRcFphbGt5V20xV2JFNXRUWGxaVkdNeldsZE9hbHBxUVRSWlZFRXhUVVJvYTFsNlZUTk9SRWswV2tSUk5FOUhWbXROUWsxSFEybHpSMEZSVVVKbk56aDNDa0ZSVVVWQ1ZVb3hZVmQ0YTAxRGMwZERhWE5IUVZGUlFtYzNPSGRCVVZWRlNGZG9NVm95WkhCaWJXUnRXVmRPYkV3eWRHeGpiVFZzWWtoTmRGa3lPWFFLWWxoV2RXRllValZOUWpCSFEybHpSMEZSVVVKbk56aDNRVkZaUlVRelNteGFiazEyWVVkV2FGcElUWFppVjBad1ltcEJOMEpuYjNKQ1owVkZRVmxQTHdwTlFVVkpRa013VFVzeWFEQmtTRUo2VDJrNGRtUkhPWEphVnpSMVdWZE9NR0ZYT1hWamVUVnVZVmhTYjJSWFNqRmpNbFo1V1RJNWRXUkhWblZrUXpWcUNtSXlNSGRpVVZsTFMzZFpRa0pCUjBSMmVrRkNRMUZTWmtSR01XOWtTRkozWTNwdmRrd3laSEJrUjJneFdXazFhbUl5TUhaaFNGWnVXakpzZFZveVdtZ0tXVEpWZG1FeVZubGliVlp6WTNreGFtSXlNWFJrVnpWd1pFaHJka3h0WkhCa1IyZ3hXV2s1TTJJelNuSmFiWGgyWkROTmRsbHVWbkJpUjFGMVpWZEdkQXBpUlVKNVdsZGFla3d5YUd4WlYxSjZUREl4YUdGWE5IZFBRVmxMUzNkWlFrSkJSMFIyZWtGQ1EyZFJjVVJEYUcxYVZFcHBXV3BaTWxwdFZteE9iVTE1Q2xsVVl6TmFWMDVxV21wQk5GbFVRVEZOUkdocldYcFZNMDVFU1RSYVJGRTBUMGRXYTAxQ2MwZERhWE5IUVZGUlFtYzNPSGRCVVhORlJGRjNUR015Vm5NS1dta3hiMkl6VGpCYVYxRjNVVUZaUzB0M1dVSkNRVWRFZG5wQlFrUkJVWGxFUkVKdlpFaFNkMk42YjNaTU1tUndaRWRvTVZscE5XcGlNakIyWVVoV2JncGFNbXgxV2pKYWFGa3lWWFpoTWxaNVltMVdjMk41TVdwaU1qRjBaRmMxY0dSSWEzZFBRVmxMUzNkWlFrSkJSMFIyZWtGQ1JGRlJjVVJEYUcxYVZFcHBDbGxxV1RKYWJWWnNUbTFOZVZsVVl6TmFWMDVxV21wQk5GbFVRVEZOUkdocldYcFZNMDVFU1RSYVJGRTBUMGRXYTAxQ09FZERhWE5IUVZGUlFtYzNPSGNLUVZFMFJVVlJkMUJqYlZadFkzazViMXBYUm10amVUbDBXVmRzZFUxQ2IwZERhWE5IUVZGUlFtYzNPSGRCVVRoRlJFRjNTMDFVUVROTlZGRXpUbFJWZVFwUFZFRjFRbWR2Y2tKblJVVkJXVTh2VFVGRlVVSkRRVTFJYldnd1pFaENlazlwT0haYU1td3dZVWhXYVV4dFRuWmlVemx2WkZka2JtRlhOVzVhYlVacUNscFVRVmxDWjI5eVFtZEZSVUZaVHk5TlFVVlNRa0Z2VFVORVNURk9la2wzVG5wUmVrMUhNRWREYVhOSFFWRlJRbWMzT0hkQlVrbEZXSGQ0WkdGSVVqQUtZMGhOTmt4NU9XNWhXRkp2WkZkSmRWa3lPWFJNTW1neFdqSmtjR0p0WkcxWlYwNXNUREowYkdOdE5XeGlTRTEwV1RJNWRHSllWblZoV0ZJMVRIazFiZ3BoV0ZKdlpGZEpkbVF5T1hsaE1scHpZak5rZWt3eVNqRmhWM2hyVEc1c2FHSlhlRUZqYlZadFkzazViMXBYUm10amVUbDBXVmRzZFUxRVowZERhWE5IQ2tGUlVVSm5OemgzUVZKTlJVdG5kMjlhYlZWNVdXMUpNazV0V214YVZGcHFUVzFGTTA0eVZtcFpNbGwzVDBkRmQwNVVRVFJhUjAweFRucFJlVTlIVVRBS1QwUm9iRnBFUVdoQ1oyOXlRbWRGUlVGWlR5OU5RVVZWUWtKTlRVVllaSFpqYlhSdFlrYzVNMWd5VW5Cak0wSm9aRWRPYjAxSFVVZERhWE5IUVZGUlFncG5OemgzUVZKVlJWWm5lRlZoU0ZJd1kwaE5Oa3g1T1c1aFdGSnZaRmRKZFZreU9YUk1NbWd4V2pKa2NHSnRaRzFaVjA1c1RESjBiR050Tld4aVNFMTBDbGt5T1hSaVdGWjFZVmhTTlV3eVJtcGtSMngyWW01TmRtTnVWblZqZVRoNlQxUkplazlVVlhoUFJFVjNUa001YUdSSVVteGlXRUl3WTNrNGVFMUNXVWNLUTJselIwRlJVVUpuTnpoM1FWSlpSVU5CZDBkalNGWnBZa2RzYWsxRldVZERhWE5IUVZGUlFtYzNPSGRCVW1kRlQwRjNNbU50Vm5kaWVuQnZaRmRrYmdwaFZ6VnVXbTFHYWxwVE9YSmFXRXAxV2xkNGVreFhUblppVnpFeFltMXNNR1ZVY0hsYVYxazJZMjFXYldONU9XOWFWMFpyWTNrNWRGbFhiSFZOU1VkTUNrSm5iM0pDWjBWRlFXUmFOVUZuVVVOQ1NEQkZaWGRDTlVGSVkwRXpWREIzWVhOaVNFVlVTbXBIVWpSamJWZGpNMEZ4U2t0WWNtcGxVRXN6TDJnMGNIa0taME00Y0Rkdk5FRkJRVWRvVm10d1JuTm5RVUZDUVUxQlUwUkNSMEZwUlVGek9XTlRVREJRTDNCU1dqUndjbUZpVGs1SmNUZzVWRk13VW1oWU1tWXpTZ3BTY0VoeWVXVkphRkl6V1VOSlVVUnVVWFYxV0hoVGJtNXVabXc0VVRkVlVrSlROMFJETWpOWmNrUmhVM0J6YUZOU1ZVVk9XSFJvVTJWNlFVdENaMmR4Q21ocmFrOVFVVkZFUVhkT2JrRkVRbXRCYWtWQk5HZERRMlV2TURCRVMxSXZOemhHYTNOU2N6RXhRbWROVjFOVGMybHphemhOUlRaUFRYbDRNbHBQUzBrS2RrMVdlVEZXU0VreE5VVTJXR1p5TTJselFTdEJhVGxhU1RaUGRIWTJRVzVQVm5jdmVsVklTMHgyZFdvMlZtOVpaRXh3TWpaVWQyaFhXVTR5VGpoTVJncGFhbnBXUjA1WUt6Qk9ZMDQ0VHpaeVdXTlFWWE4zUFQwS0xTMHRMUzFGVGtRZ1EwVlNWRWxHU1VOQlZFVXRMUzB0TFFvPSJ9fX19"}], "timestampVerificationData":{"rfc3161Timestamps":[{"signedTimestamp":"MIICyDADAgEAMIICvwYJKoZIhvcNAQcCoIICsDCCAqwCAQMxDTALBglghkgBZQMEAgEwgbcGCyqGSIb3DQEJEAEEoIGnBIGkMIGhAgEBBgkrBgEEAYO/MAIwMTANBglghkgBZQMEAgEFAAQgVRiPQiyjqHesLhjA1G/TxjESiy/FW8Sle/1OYNu3K/kCFAwvXO0pPiaW4uamLJqBsr3bixmnGA8yMDI2MTAxOTIyMzExMFowAwIBAaAypDAwLjEVMBMGA1UEChMMc2lnc3RvcmUuZGV2MRUwEwYDVQQDEwxzaWdzdG9yZS10c2GgADGCAdowggHWAgEBMFEwOTEVMBMGA1UEChMMc2lnc3RvcmUuZGV2MSAwHgYDVQQDExdzaWdzdG9yZS10c2Etc2VsZnNpZ25lZAIUOhNULwyQYe68wUMvy4qOiyojiwwwCwYJYIZIAWUDBAIBoIH8MBoGCSqGSIb3DQEJAzENBgsqhkiG9w0BCRABBDAcBgkqhkiG9w0BCQUxDxcNMjYxMDE5MjIzMTEwWjAvBgkqhkiG9w0BCQQxIgQgMtOdPWovwN2MtCUsGr55eXS/sj80XwnkzhWzvp3Z+ZYwgY4GCyqGSIb3DQEJEAIvMX8wfTB7MHkEIIX5J7wHq2LKw7RDVsEO/IGyxog/2nq55thw2dE6zQW3MFUwPaQ7MDkxFTATBgNVBAoTDHNpZ3N0b3JlLmRldjEgMB4GA1UEAxMXc2lnc3RvcmUtdHNhLXNlbGZzaWduZWQCFDoTVC8MkGHuvMFDL8uKjosqI4sMMAoGCCqGSM49BAMCBGYwZAIwa5hPScsoY8ZaoBk5cLEAUwhE+yEEOwm9xprfopTKz7qKFY1O9DeK5h6/gp2k6NtgAjByqi8mda8A+OGfG6rtkFHgjYnT6P/VUqQ95DNedeBl+vbMOity7vq28Gh+8O2yqBc="}]}}, "messageSignature":{"messageDigest":{"algorithm":"SHA2_256", "digest":"CPoM1QqE+UEnbF5tX6SfHU4B4VmHpgqrjyq24xnux3s="}, "signature":"MEUCICPSiaX9pY/oZbzuasGAnOQ0uL8P/JLHK+FKmUHYfvT2AiEAs8FC8HRyVEfj7AGjbsI6VfoahccKXs4FCfcyngTsQXc="}}
|
build/torch210-rocm70-x86_64-linux/mha.py
ADDED
|
@@ -0,0 +1,332 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""FlashAttention forward entry points backed by aiter's Composable-Kernel FMHA.
|
| 2 |
+
|
| 3 |
+
This is the compiled (HIP / Composable Kernel) counterpart to the Triton
|
| 4 |
+
``aiter-flash-attn`` package. Only the *forward* pass is provided (dense and
|
| 5 |
+
variable-length), including support for learnable attention sinks via
|
| 6 |
+
``window_size[2]`` / ``s_aux`` (e.g. gpt-oss). The op set ships exclusively
|
| 7 |
+
the CK ``mha_fwd`` / ``mha_varlen_fwd`` kernels, so the sink path is always
|
| 8 |
+
taken (no fall-through to a non-sink ASM kernel).
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
from typing import Optional, Tuple, Union
|
| 12 |
+
|
| 13 |
+
import torch
|
| 14 |
+
from torch import Tensor
|
| 15 |
+
|
| 16 |
+
from ._ops import ops
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def _round_multiple(x: int, m: int) -> int:
|
| 20 |
+
return (x + m - 1) // m * m
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def _maybe_pad_headdim(t: Tensor, target: int) -> Tensor:
|
| 24 |
+
og = t.size(-1)
|
| 25 |
+
if og == target:
|
| 26 |
+
return t
|
| 27 |
+
return torch.nn.functional.pad(t, [0, target - og])
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def _unpack_window(window_size) -> Tuple[int, int, int]:
|
| 31 |
+
left = int(window_size[0])
|
| 32 |
+
right = int(window_size[1])
|
| 33 |
+
sink = int(window_size[2]) if len(window_size) == 3 else 0
|
| 34 |
+
return left, right, sink
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def _prep_sink(s_aux: Optional[Tensor], nheads: int, device) -> Optional[Tensor]:
|
| 38 |
+
if s_aux is None:
|
| 39 |
+
return None
|
| 40 |
+
assert s_aux.device == device, "s_aux must be on the same device as q"
|
| 41 |
+
assert s_aux.shape[0] == nheads, "s_aux must have shape [nheads_q]"
|
| 42 |
+
if s_aux.dtype != torch.float32:
|
| 43 |
+
s_aux = s_aux.to(torch.float32)
|
| 44 |
+
return s_aux
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def _effective_sink_size(sink_size: int, s_aux: Optional[Tensor]) -> int:
|
| 48 |
+
# The CK kernel only takes the sink path when sink_size > 0. Callers using
|
| 49 |
+
# the transformers attention interface pass the per-head sink logits as
|
| 50 |
+
# ``s_aux`` but leave ``window_size`` a 2-tuple (sink_size == 0), so enable
|
| 51 |
+
# a single learnable sink automatically when s_aux is given.
|
| 52 |
+
if s_aux is not None and sink_size == 0:
|
| 53 |
+
return 1
|
| 54 |
+
return sink_size
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def flash_attn_func(
|
| 58 |
+
q: Tensor,
|
| 59 |
+
k: Tensor,
|
| 60 |
+
v: Tensor,
|
| 61 |
+
dropout_p: float = 0.0,
|
| 62 |
+
softmax_scale: Optional[float] = None,
|
| 63 |
+
causal: bool = False,
|
| 64 |
+
window_size: Tuple[int, ...] = (-1, -1, 0),
|
| 65 |
+
bias: Optional[Tensor] = None,
|
| 66 |
+
alibi_slopes: Optional[Tensor] = None,
|
| 67 |
+
return_lse: bool = False,
|
| 68 |
+
return_attn_probs: bool = False,
|
| 69 |
+
s_aux: Optional[Tensor] = None,
|
| 70 |
+
**kwargs,
|
| 71 |
+
) -> Union[Tensor, Tuple[Tensor, ...]]:
|
| 72 |
+
"""Dense FlashAttention forward.
|
| 73 |
+
|
| 74 |
+
``q, k, v`` have shape ``(batch, seqlen, nheads, headdim)``. ``window_size``
|
| 75 |
+
is ``(left, right, sink_size)``; ``s_aux`` is an optional ``(nheads_q,)``
|
| 76 |
+
fp32 tensor of learnable sink logits. Extra keyword arguments (e.g. those
|
| 77 |
+
passed by the transformers flash-attention interface) are ignored.
|
| 78 |
+
"""
|
| 79 |
+
if softmax_scale is None:
|
| 80 |
+
softmax_scale = q.shape[-1] ** (-0.5)
|
| 81 |
+
window_size_left, window_size_right, sink_size = _unpack_window(window_size)
|
| 82 |
+
|
| 83 |
+
nheads_q = q.size(2)
|
| 84 |
+
s_aux = _prep_sink(s_aux, nheads_q, q.device)
|
| 85 |
+
sink_size = _effective_sink_size(sink_size, s_aux)
|
| 86 |
+
|
| 87 |
+
head_q_og = q.size(-1)
|
| 88 |
+
head_v_og = v.size(-1)
|
| 89 |
+
head_q = _round_multiple(head_q_og, 8)
|
| 90 |
+
head_v = _round_multiple(head_v_og, 8)
|
| 91 |
+
q = _maybe_pad_headdim(q, head_q)
|
| 92 |
+
k = _maybe_pad_headdim(k, head_q)
|
| 93 |
+
v = _maybe_pad_headdim(v, head_v)
|
| 94 |
+
|
| 95 |
+
return_softmax = return_attn_probs and dropout_p > 0
|
| 96 |
+
out, softmax_lse, S_dmask, _ = ops.mha_fwd(
|
| 97 |
+
q,
|
| 98 |
+
k,
|
| 99 |
+
v,
|
| 100 |
+
dropout_p,
|
| 101 |
+
softmax_scale,
|
| 102 |
+
causal,
|
| 103 |
+
window_size_left,
|
| 104 |
+
window_size_right,
|
| 105 |
+
sink_size,
|
| 106 |
+
return_lse or return_attn_probs,
|
| 107 |
+
return_softmax,
|
| 108 |
+
None, # cu_seqlens_q
|
| 109 |
+
None, # cu_seqlens_kv
|
| 110 |
+
None, # out
|
| 111 |
+
bias,
|
| 112 |
+
alibi_slopes,
|
| 113 |
+
None, # q_descale
|
| 114 |
+
None, # k_descale
|
| 115 |
+
None, # v_descale
|
| 116 |
+
s_aux,
|
| 117 |
+
None, # gen
|
| 118 |
+
)
|
| 119 |
+
|
| 120 |
+
out = out[..., :head_v_og]
|
| 121 |
+
if not (return_lse or return_attn_probs):
|
| 122 |
+
return out
|
| 123 |
+
result = (out,)
|
| 124 |
+
if return_lse:
|
| 125 |
+
result = result + (softmax_lse,)
|
| 126 |
+
if return_attn_probs:
|
| 127 |
+
result = result + (S_dmask,)
|
| 128 |
+
return result
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
def flash_attn_varlen_func(
|
| 132 |
+
q: Tensor,
|
| 133 |
+
k: Tensor,
|
| 134 |
+
v: Tensor,
|
| 135 |
+
cu_seqlens_q: Tensor,
|
| 136 |
+
cu_seqlens_k: Tensor,
|
| 137 |
+
max_seqlen_q: int,
|
| 138 |
+
max_seqlen_k: int,
|
| 139 |
+
dropout_p: float = 0.0,
|
| 140 |
+
softmax_scale: Optional[float] = None,
|
| 141 |
+
causal: bool = False,
|
| 142 |
+
window_size: Tuple[int, ...] = (-1, -1, 0),
|
| 143 |
+
bias: Optional[Tensor] = None,
|
| 144 |
+
alibi_slopes: Optional[Tensor] = None,
|
| 145 |
+
return_lse: bool = False,
|
| 146 |
+
return_attn_probs: bool = False,
|
| 147 |
+
block_table: Optional[Tensor] = None,
|
| 148 |
+
min_seqlen_q: int = 0,
|
| 149 |
+
logits_soft_cap: float = 0.0,
|
| 150 |
+
zero_tensors: bool = False,
|
| 151 |
+
cu_seqlens_q_padded: Optional[Tensor] = None,
|
| 152 |
+
cu_seqlens_k_padded: Optional[Tensor] = None,
|
| 153 |
+
s_aux: Optional[Tensor] = None,
|
| 154 |
+
**kwargs,
|
| 155 |
+
) -> Union[Tensor, Tuple[Tensor, ...]]:
|
| 156 |
+
"""Variable-length / packed FlashAttention forward.
|
| 157 |
+
|
| 158 |
+
``q`` has shape ``(total_q, nheads, headdim)``; ``cu_seqlens_*`` are int32
|
| 159 |
+
cumulative offsets of shape ``(batch + 1,)``. Sinks are supported through
|
| 160 |
+
``window_size[2]`` / ``s_aux`` as in :func:`flash_attn_func`. Extra keyword
|
| 161 |
+
arguments (e.g. from the transformers flash-attention interface) are ignored.
|
| 162 |
+
"""
|
| 163 |
+
if softmax_scale is None:
|
| 164 |
+
softmax_scale = q.shape[-1] ** (-0.5)
|
| 165 |
+
window_size_left, window_size_right, sink_size = _unpack_window(window_size)
|
| 166 |
+
|
| 167 |
+
nheads_q = q.size(1)
|
| 168 |
+
s_aux = _prep_sink(s_aux, nheads_q, q.device)
|
| 169 |
+
sink_size = _effective_sink_size(sink_size, s_aux)
|
| 170 |
+
|
| 171 |
+
head_q_og = q.size(-1)
|
| 172 |
+
head_v_og = v.size(-1)
|
| 173 |
+
head_q = _round_multiple(head_q_og, 8)
|
| 174 |
+
head_v = _round_multiple(head_v_og, 8)
|
| 175 |
+
q = _maybe_pad_headdim(q, head_q)
|
| 176 |
+
k = _maybe_pad_headdim(k, head_q)
|
| 177 |
+
v = _maybe_pad_headdim(v, head_v)
|
| 178 |
+
|
| 179 |
+
return_softmax = return_attn_probs and dropout_p > 0
|
| 180 |
+
out, softmax_lse, S_dmask, _ = ops.mha_varlen_fwd(
|
| 181 |
+
q,
|
| 182 |
+
k,
|
| 183 |
+
v,
|
| 184 |
+
cu_seqlens_q,
|
| 185 |
+
cu_seqlens_k,
|
| 186 |
+
max_seqlen_q,
|
| 187 |
+
max_seqlen_k,
|
| 188 |
+
min_seqlen_q,
|
| 189 |
+
dropout_p,
|
| 190 |
+
softmax_scale,
|
| 191 |
+
logits_soft_cap,
|
| 192 |
+
zero_tensors,
|
| 193 |
+
causal,
|
| 194 |
+
window_size_left,
|
| 195 |
+
window_size_right,
|
| 196 |
+
sink_size,
|
| 197 |
+
return_lse or return_attn_probs,
|
| 198 |
+
return_softmax,
|
| 199 |
+
None, # out
|
| 200 |
+
block_table,
|
| 201 |
+
bias,
|
| 202 |
+
alibi_slopes,
|
| 203 |
+
None, # q_descale
|
| 204 |
+
None, # k_descale
|
| 205 |
+
None, # v_descale
|
| 206 |
+
None, # gen
|
| 207 |
+
cu_seqlens_q_padded,
|
| 208 |
+
cu_seqlens_k_padded,
|
| 209 |
+
s_aux,
|
| 210 |
+
)
|
| 211 |
+
|
| 212 |
+
out = out[..., :head_v_og]
|
| 213 |
+
if not (return_lse or return_attn_probs):
|
| 214 |
+
return out
|
| 215 |
+
result = (out,)
|
| 216 |
+
if return_lse:
|
| 217 |
+
result = result + (softmax_lse,)
|
| 218 |
+
if return_attn_probs:
|
| 219 |
+
result = result + (S_dmask,)
|
| 220 |
+
return result
|
| 221 |
+
|
| 222 |
+
|
| 223 |
+
def _append_to_contiguous_cache(k_cache, v_cache, k, v, cache_seqlens, Sq):
|
| 224 |
+
"""In-place append of new k/v into a contiguous cache at per-batch offsets."""
|
| 225 |
+
B = k_cache.shape[0]
|
| 226 |
+
if isinstance(cache_seqlens, int):
|
| 227 |
+
cache_seqlens = torch.full((B,), cache_seqlens, device=k_cache.device, dtype=torch.long)
|
| 228 |
+
pos = cache_seqlens.view(B, 1).to(torch.long) + torch.arange(Sq, device=k_cache.device).view(1, Sq)
|
| 229 |
+
bidx = torch.arange(B, device=k_cache.device).view(B, 1).expand(B, Sq)
|
| 230 |
+
k_cache[bidx, pos] = k.to(k_cache.dtype)
|
| 231 |
+
v_cache[bidx, pos] = v.to(v_cache.dtype)
|
| 232 |
+
|
| 233 |
+
|
| 234 |
+
def flash_attn_with_kvcache(
|
| 235 |
+
q: Tensor,
|
| 236 |
+
k_cache: Tensor,
|
| 237 |
+
v_cache: Tensor,
|
| 238 |
+
k: Optional[Tensor] = None,
|
| 239 |
+
v: Optional[Tensor] = None,
|
| 240 |
+
rotary_cos: Optional[Tensor] = None,
|
| 241 |
+
rotary_sin: Optional[Tensor] = None,
|
| 242 |
+
cache_seqlens=None,
|
| 243 |
+
block_table: Optional[Tensor] = None,
|
| 244 |
+
softmax_scale: Optional[float] = None,
|
| 245 |
+
causal: bool = True,
|
| 246 |
+
window_size: Tuple[int, ...] = (-1, -1, 0),
|
| 247 |
+
s_aux: Optional[Tensor] = None,
|
| 248 |
+
return_lse: bool = False,
|
| 249 |
+
**kwargs,
|
| 250 |
+
) -> Union[Tensor, Tuple[Tensor, ...]]:
|
| 251 |
+
"""FlashAttention against a KV cache, built on the CK split-KV kernel.
|
| 252 |
+
|
| 253 |
+
``q`` is ``(batch, seqlen_q, nheads, headdim)``. For a contiguous cache,
|
| 254 |
+
``k_cache``/``v_cache`` are ``(batch, seqlen_cache, nheads_k, headdim)`` and
|
| 255 |
+
``cache_seqlens`` (int or ``(batch,)``) is the number of valid cache entries
|
| 256 |
+
*before* appending. If ``k``/``v`` (the new tokens) are given, they are
|
| 257 |
+
written into the cache in place. The query attends to the valid prefix
|
| 258 |
+
(``cache_seqlens`` + ``seqlen_q``) with bottom-right causal alignment.
|
| 259 |
+
|
| 260 |
+
Notes / current limitations:
|
| 261 |
+
- Rotary embedding is not applied here; pass ``q``/``k`` already rotated
|
| 262 |
+
(``rotary_cos``/``rotary_sin`` are rejected).
|
| 263 |
+
- Paged caches are addressed via ``block_table`` (forwarded to the split-KV
|
| 264 |
+
varlen kernel); in-cache append for paged layouts must be done by the
|
| 265 |
+
caller (pass ``k=v=None``).
|
| 266 |
+
"""
|
| 267 |
+
if rotary_cos is not None or rotary_sin is not None:
|
| 268 |
+
raise NotImplementedError(
|
| 269 |
+
"flash_attn_with_kvcache does not apply rotary; rotate q/k beforehand."
|
| 270 |
+
)
|
| 271 |
+
if softmax_scale is None:
|
| 272 |
+
softmax_scale = q.shape[-1] ** (-0.5)
|
| 273 |
+
window_size_left, window_size_right, sink_size = _unpack_window(window_size)
|
| 274 |
+
B, Sq, Hq, D = q.shape
|
| 275 |
+
s_aux = _prep_sink(s_aux, Hq, q.device)
|
| 276 |
+
sink_size = _effective_sink_size(sink_size, s_aux)
|
| 277 |
+
|
| 278 |
+
paged = block_table is not None
|
| 279 |
+
|
| 280 |
+
# ---- append new tokens into the cache (contiguous layout only) ----
|
| 281 |
+
if k is not None and not paged:
|
| 282 |
+
assert cache_seqlens is not None, "cache_seqlens required to append into the cache"
|
| 283 |
+
_append_to_contiguous_cache(k_cache, v_cache, k, v, cache_seqlens, Sq)
|
| 284 |
+
elif k is not None and paged:
|
| 285 |
+
raise NotImplementedError(
|
| 286 |
+
"in-cache append for paged layouts is not supported; append externally "
|
| 287 |
+
"and call with k=v=None."
|
| 288 |
+
)
|
| 289 |
+
|
| 290 |
+
appended = 0 if k is None else Sq
|
| 291 |
+
|
| 292 |
+
# ---- key lengths ----
|
| 293 |
+
if cache_seqlens is None:
|
| 294 |
+
klens = torch.full((B,), k_cache.shape[1], device=q.device, dtype=torch.long)
|
| 295 |
+
elif isinstance(cache_seqlens, int):
|
| 296 |
+
klens = torch.full((B,), cache_seqlens + appended, device=q.device, dtype=torch.long)
|
| 297 |
+
else:
|
| 298 |
+
klens = cache_seqlens.to(q.device, torch.long) + appended
|
| 299 |
+
|
| 300 |
+
uniform = bool((klens == klens[0]).all().item())
|
| 301 |
+
|
| 302 |
+
# ---- fast path: contiguous cache, uniform length -> dense kernel ----
|
| 303 |
+
if not paged and uniform:
|
| 304 |
+
L = int(klens[0].item())
|
| 305 |
+
return flash_attn_func(
|
| 306 |
+
q, k_cache[:, :L], v_cache[:, :L],
|
| 307 |
+
softmax_scale=softmax_scale, causal=causal,
|
| 308 |
+
window_size=(window_size_left, window_size_right, sink_size),
|
| 309 |
+
s_aux=s_aux, return_lse=return_lse,
|
| 310 |
+
)
|
| 311 |
+
|
| 312 |
+
# ---- general path: variable cache lengths -> split-KV varlen kernel ----
|
| 313 |
+
cu_q = torch.arange(0, (B + 1) * Sq, Sq, device=q.device, dtype=torch.int32)
|
| 314 |
+
cu_k = torch.zeros(B + 1, device=q.device, dtype=torch.int32)
|
| 315 |
+
cu_k[1:] = torch.cumsum(klens, 0).to(torch.int32)
|
| 316 |
+
q_packed = q.reshape(B * Sq, Hq, D)
|
| 317 |
+
if paged:
|
| 318 |
+
k_packed, v_packed = k_cache, v_cache # kernel gathers via block_table
|
| 319 |
+
else:
|
| 320 |
+
k_packed = torch.cat([k_cache[b, : int(klens[b])] for b in range(B)], dim=0)
|
| 321 |
+
v_packed = torch.cat([v_cache[b, : int(klens[b])] for b in range(B)], dim=0)
|
| 322 |
+
|
| 323 |
+
out = flash_attn_varlen_func(
|
| 324 |
+
q_packed, k_packed, v_packed, cu_q, cu_k, Sq, int(klens.max().item()),
|
| 325 |
+
softmax_scale=softmax_scale, causal=causal,
|
| 326 |
+
window_size=(window_size_left, window_size_right, sink_size),
|
| 327 |
+
block_table=block_table, s_aux=s_aux, return_lse=return_lse,
|
| 328 |
+
)
|
| 329 |
+
if return_lse:
|
| 330 |
+
out, lse = out
|
| 331 |
+
return out.reshape(B, Sq, Hq, D), lse
|
| 332 |
+
return out.reshape(B, Sq, Hq, D)
|
build/torch210-rocm70-x86_64-linux/symbols.json
ADDED
|
@@ -0,0 +1,349 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": 1,
|
| 3 |
+
"module": "_aiter_flash_attn_ck_rocm_fe2bb66",
|
| 4 |
+
"functions": [
|
| 5 |
+
{
|
| 6 |
+
"name": "flash_attn_func",
|
| 7 |
+
"qualname": "flash_attn_func",
|
| 8 |
+
"module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha",
|
| 9 |
+
"kind": "function",
|
| 10 |
+
"doc": "Dense FlashAttention forward.\n\n``q, k, v`` have shape ``(batch, seqlen, nheads, headdim)``. ``window_size``\nis ``(left, right, sink_size)``; ``s_aux`` is an optional ``(nheads_q,)``\nfp32 tensor of learnable sink logits. Extra keyword arguments (e.g. those\npassed by the transformers flash-attention interface) are ignored.",
|
| 11 |
+
"signature": {
|
| 12 |
+
"parameters": [
|
| 13 |
+
{
|
| 14 |
+
"name": "q",
|
| 15 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 16 |
+
"default": null,
|
| 17 |
+
"annotation": "torch.Tensor"
|
| 18 |
+
},
|
| 19 |
+
{
|
| 20 |
+
"name": "k",
|
| 21 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 22 |
+
"default": null,
|
| 23 |
+
"annotation": "torch.Tensor"
|
| 24 |
+
},
|
| 25 |
+
{
|
| 26 |
+
"name": "v",
|
| 27 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 28 |
+
"default": null,
|
| 29 |
+
"annotation": "torch.Tensor"
|
| 30 |
+
},
|
| 31 |
+
{
|
| 32 |
+
"name": "dropout_p",
|
| 33 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 34 |
+
"default": "0.0",
|
| 35 |
+
"annotation": "float"
|
| 36 |
+
},
|
| 37 |
+
{
|
| 38 |
+
"name": "softmax_scale",
|
| 39 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 40 |
+
"default": "None",
|
| 41 |
+
"annotation": "float | None"
|
| 42 |
+
},
|
| 43 |
+
{
|
| 44 |
+
"name": "causal",
|
| 45 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 46 |
+
"default": "False",
|
| 47 |
+
"annotation": "bool"
|
| 48 |
+
},
|
| 49 |
+
{
|
| 50 |
+
"name": "window_size",
|
| 51 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 52 |
+
"default": "(-1, -1, 0)",
|
| 53 |
+
"annotation": "Tuple[int, ...]"
|
| 54 |
+
},
|
| 55 |
+
{
|
| 56 |
+
"name": "bias",
|
| 57 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 58 |
+
"default": "None",
|
| 59 |
+
"annotation": "torch.Tensor | None"
|
| 60 |
+
},
|
| 61 |
+
{
|
| 62 |
+
"name": "alibi_slopes",
|
| 63 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 64 |
+
"default": "None",
|
| 65 |
+
"annotation": "torch.Tensor | None"
|
| 66 |
+
},
|
| 67 |
+
{
|
| 68 |
+
"name": "return_lse",
|
| 69 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 70 |
+
"default": "False",
|
| 71 |
+
"annotation": "bool"
|
| 72 |
+
},
|
| 73 |
+
{
|
| 74 |
+
"name": "return_attn_probs",
|
| 75 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 76 |
+
"default": "False",
|
| 77 |
+
"annotation": "bool"
|
| 78 |
+
},
|
| 79 |
+
{
|
| 80 |
+
"name": "s_aux",
|
| 81 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 82 |
+
"default": "None",
|
| 83 |
+
"annotation": "torch.Tensor | None"
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"name": "kwargs",
|
| 87 |
+
"kind": "VAR_KEYWORD",
|
| 88 |
+
"default": null,
|
| 89 |
+
"annotation": null
|
| 90 |
+
}
|
| 91 |
+
],
|
| 92 |
+
"return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]"
|
| 93 |
+
}
|
| 94 |
+
},
|
| 95 |
+
{
|
| 96 |
+
"name": "flash_attn_varlen_func",
|
| 97 |
+
"qualname": "flash_attn_varlen_func",
|
| 98 |
+
"module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha",
|
| 99 |
+
"kind": "function",
|
| 100 |
+
"doc": "Variable-length / packed FlashAttention forward.\n\n``q`` has shape ``(total_q, nheads, headdim)``; ``cu_seqlens_*`` are int32\ncumulative offsets of shape ``(batch + 1,)``. Sinks are supported through\n``window_size[2]`` / ``s_aux`` as in :func:`flash_attn_func`. Extra keyword\narguments (e.g. from the transformers flash-attention interface) are ignored.",
|
| 101 |
+
"signature": {
|
| 102 |
+
"parameters": [
|
| 103 |
+
{
|
| 104 |
+
"name": "q",
|
| 105 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 106 |
+
"default": null,
|
| 107 |
+
"annotation": "torch.Tensor"
|
| 108 |
+
},
|
| 109 |
+
{
|
| 110 |
+
"name": "k",
|
| 111 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 112 |
+
"default": null,
|
| 113 |
+
"annotation": "torch.Tensor"
|
| 114 |
+
},
|
| 115 |
+
{
|
| 116 |
+
"name": "v",
|
| 117 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 118 |
+
"default": null,
|
| 119 |
+
"annotation": "torch.Tensor"
|
| 120 |
+
},
|
| 121 |
+
{
|
| 122 |
+
"name": "cu_seqlens_q",
|
| 123 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 124 |
+
"default": null,
|
| 125 |
+
"annotation": "torch.Tensor"
|
| 126 |
+
},
|
| 127 |
+
{
|
| 128 |
+
"name": "cu_seqlens_k",
|
| 129 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 130 |
+
"default": null,
|
| 131 |
+
"annotation": "torch.Tensor"
|
| 132 |
+
},
|
| 133 |
+
{
|
| 134 |
+
"name": "max_seqlen_q",
|
| 135 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 136 |
+
"default": null,
|
| 137 |
+
"annotation": "int"
|
| 138 |
+
},
|
| 139 |
+
{
|
| 140 |
+
"name": "max_seqlen_k",
|
| 141 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 142 |
+
"default": null,
|
| 143 |
+
"annotation": "int"
|
| 144 |
+
},
|
| 145 |
+
{
|
| 146 |
+
"name": "dropout_p",
|
| 147 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 148 |
+
"default": "0.0",
|
| 149 |
+
"annotation": "float"
|
| 150 |
+
},
|
| 151 |
+
{
|
| 152 |
+
"name": "softmax_scale",
|
| 153 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 154 |
+
"default": "None",
|
| 155 |
+
"annotation": "float | None"
|
| 156 |
+
},
|
| 157 |
+
{
|
| 158 |
+
"name": "causal",
|
| 159 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 160 |
+
"default": "False",
|
| 161 |
+
"annotation": "bool"
|
| 162 |
+
},
|
| 163 |
+
{
|
| 164 |
+
"name": "window_size",
|
| 165 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 166 |
+
"default": "(-1, -1, 0)",
|
| 167 |
+
"annotation": "Tuple[int, ...]"
|
| 168 |
+
},
|
| 169 |
+
{
|
| 170 |
+
"name": "bias",
|
| 171 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 172 |
+
"default": "None",
|
| 173 |
+
"annotation": "torch.Tensor | None"
|
| 174 |
+
},
|
| 175 |
+
{
|
| 176 |
+
"name": "alibi_slopes",
|
| 177 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 178 |
+
"default": "None",
|
| 179 |
+
"annotation": "torch.Tensor | None"
|
| 180 |
+
},
|
| 181 |
+
{
|
| 182 |
+
"name": "return_lse",
|
| 183 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 184 |
+
"default": "False",
|
| 185 |
+
"annotation": "bool"
|
| 186 |
+
},
|
| 187 |
+
{
|
| 188 |
+
"name": "return_attn_probs",
|
| 189 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 190 |
+
"default": "False",
|
| 191 |
+
"annotation": "bool"
|
| 192 |
+
},
|
| 193 |
+
{
|
| 194 |
+
"name": "block_table",
|
| 195 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 196 |
+
"default": "None",
|
| 197 |
+
"annotation": "torch.Tensor | None"
|
| 198 |
+
},
|
| 199 |
+
{
|
| 200 |
+
"name": "min_seqlen_q",
|
| 201 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 202 |
+
"default": "0",
|
| 203 |
+
"annotation": "int"
|
| 204 |
+
},
|
| 205 |
+
{
|
| 206 |
+
"name": "logits_soft_cap",
|
| 207 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 208 |
+
"default": "0.0",
|
| 209 |
+
"annotation": "float"
|
| 210 |
+
},
|
| 211 |
+
{
|
| 212 |
+
"name": "zero_tensors",
|
| 213 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 214 |
+
"default": "False",
|
| 215 |
+
"annotation": "bool"
|
| 216 |
+
},
|
| 217 |
+
{
|
| 218 |
+
"name": "cu_seqlens_q_padded",
|
| 219 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 220 |
+
"default": "None",
|
| 221 |
+
"annotation": "torch.Tensor | None"
|
| 222 |
+
},
|
| 223 |
+
{
|
| 224 |
+
"name": "cu_seqlens_k_padded",
|
| 225 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 226 |
+
"default": "None",
|
| 227 |
+
"annotation": "torch.Tensor | None"
|
| 228 |
+
},
|
| 229 |
+
{
|
| 230 |
+
"name": "s_aux",
|
| 231 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 232 |
+
"default": "None",
|
| 233 |
+
"annotation": "torch.Tensor | None"
|
| 234 |
+
},
|
| 235 |
+
{
|
| 236 |
+
"name": "kwargs",
|
| 237 |
+
"kind": "VAR_KEYWORD",
|
| 238 |
+
"default": null,
|
| 239 |
+
"annotation": null
|
| 240 |
+
}
|
| 241 |
+
],
|
| 242 |
+
"return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]"
|
| 243 |
+
}
|
| 244 |
+
},
|
| 245 |
+
{
|
| 246 |
+
"name": "flash_attn_with_kvcache",
|
| 247 |
+
"qualname": "flash_attn_with_kvcache",
|
| 248 |
+
"module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha",
|
| 249 |
+
"kind": "function",
|
| 250 |
+
"doc": "FlashAttention against a KV cache, built on the CK split-KV kernel.\n\n``q`` is ``(batch, seqlen_q, nheads, headdim)``. For a contiguous cache,\n``k_cache``/``v_cache`` are ``(batch, seqlen_cache, nheads_k, headdim)`` and\n``cache_seqlens`` (int or ``(batch,)``) is the number of valid cache entries\n*before* appending. If ``k``/``v`` (the new tokens) are given, they are\nwritten into the cache in place. The query attends to the valid prefix\n(``cache_seqlens`` + ``seqlen_q``) with bottom-right causal alignment.\n\nNotes / current limitations:\n- Rotary embedding is not applied here; pass ``q``/``k`` already rotated\n (``rotary_cos``/``rotary_sin`` are rejected).\n- Paged caches are addressed via ``block_table`` (forwarded to the split-KV\n varlen kernel); in-cache append for paged layouts must be done by the\n caller (pass ``k=v=None``).",
|
| 251 |
+
"signature": {
|
| 252 |
+
"parameters": [
|
| 253 |
+
{
|
| 254 |
+
"name": "q",
|
| 255 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 256 |
+
"default": null,
|
| 257 |
+
"annotation": "torch.Tensor"
|
| 258 |
+
},
|
| 259 |
+
{
|
| 260 |
+
"name": "k_cache",
|
| 261 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 262 |
+
"default": null,
|
| 263 |
+
"annotation": "torch.Tensor"
|
| 264 |
+
},
|
| 265 |
+
{
|
| 266 |
+
"name": "v_cache",
|
| 267 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 268 |
+
"default": null,
|
| 269 |
+
"annotation": "torch.Tensor"
|
| 270 |
+
},
|
| 271 |
+
{
|
| 272 |
+
"name": "k",
|
| 273 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 274 |
+
"default": "None",
|
| 275 |
+
"annotation": "torch.Tensor | None"
|
| 276 |
+
},
|
| 277 |
+
{
|
| 278 |
+
"name": "v",
|
| 279 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 280 |
+
"default": "None",
|
| 281 |
+
"annotation": "torch.Tensor | None"
|
| 282 |
+
},
|
| 283 |
+
{
|
| 284 |
+
"name": "rotary_cos",
|
| 285 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 286 |
+
"default": "None",
|
| 287 |
+
"annotation": "torch.Tensor | None"
|
| 288 |
+
},
|
| 289 |
+
{
|
| 290 |
+
"name": "rotary_sin",
|
| 291 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 292 |
+
"default": "None",
|
| 293 |
+
"annotation": "torch.Tensor | None"
|
| 294 |
+
},
|
| 295 |
+
{
|
| 296 |
+
"name": "cache_seqlens",
|
| 297 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 298 |
+
"default": "None",
|
| 299 |
+
"annotation": null
|
| 300 |
+
},
|
| 301 |
+
{
|
| 302 |
+
"name": "block_table",
|
| 303 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 304 |
+
"default": "None",
|
| 305 |
+
"annotation": "torch.Tensor | None"
|
| 306 |
+
},
|
| 307 |
+
{
|
| 308 |
+
"name": "softmax_scale",
|
| 309 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 310 |
+
"default": "None",
|
| 311 |
+
"annotation": "float | None"
|
| 312 |
+
},
|
| 313 |
+
{
|
| 314 |
+
"name": "causal",
|
| 315 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 316 |
+
"default": "True",
|
| 317 |
+
"annotation": "bool"
|
| 318 |
+
},
|
| 319 |
+
{
|
| 320 |
+
"name": "window_size",
|
| 321 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 322 |
+
"default": "(-1, -1, 0)",
|
| 323 |
+
"annotation": "Tuple[int, ...]"
|
| 324 |
+
},
|
| 325 |
+
{
|
| 326 |
+
"name": "s_aux",
|
| 327 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 328 |
+
"default": "None",
|
| 329 |
+
"annotation": "torch.Tensor | None"
|
| 330 |
+
},
|
| 331 |
+
{
|
| 332 |
+
"name": "return_lse",
|
| 333 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 334 |
+
"default": "False",
|
| 335 |
+
"annotation": "bool"
|
| 336 |
+
},
|
| 337 |
+
{
|
| 338 |
+
"name": "kwargs",
|
| 339 |
+
"kind": "VAR_KEYWORD",
|
| 340 |
+
"default": null,
|
| 341 |
+
"annotation": null
|
| 342 |
+
}
|
| 343 |
+
],
|
| 344 |
+
"return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]"
|
| 345 |
+
}
|
| 346 |
+
}
|
| 347 |
+
],
|
| 348 |
+
"layers": []
|
| 349 |
+
}
|
build/torch210-rocm71-x86_64-linux/__init__.py
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""AITER Composable-Kernel FlashAttention kernels for AMD ROCm (compiled HIP).
|
| 2 |
+
|
| 3 |
+
Compiled (Composable Kernel / HIP) counterpart to the Triton-based
|
| 4 |
+
``aiter-flash-attn`` package, repackaged from the CK FMHA forward path of the
|
| 5 |
+
`ROCm/aiter <https://github.com/ROCm/aiter>`_ project as a self-contained
|
| 6 |
+
Hugging Face Hub kernel. Exposes the FlashAttention forward entry points with
|
| 7 |
+
learnable attention sink support (e.g. gpt-oss).
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
from .mha import (
|
| 11 |
+
flash_attn_func,
|
| 12 |
+
flash_attn_varlen_func,
|
| 13 |
+
flash_attn_with_kvcache,
|
| 14 |
+
)
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
__kernel_metadata__ = {
|
| 18 |
+
"license": "mit",
|
| 19 |
+
}
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
__all__ = [
|
| 23 |
+
"__kernel_metadata__",
|
| 24 |
+
"flash_attn_func",
|
| 25 |
+
"flash_attn_varlen_func",
|
| 26 |
+
"flash_attn_with_kvcache",
|
| 27 |
+
]
|
build/torch210-rocm71-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:04aebc20ca3de88cd0cb137fba3aaf0aa8b1ccd16588df660c70c8491653f01c
|
| 3 |
+
size 204259752
|
build/torch210-rocm71-x86_64-linux/_ops.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
from . import _aiter_flash_attn_ck_rocm_fe2bb66
|
| 3 |
+
ops = torch.ops._aiter_flash_attn_ck_rocm_fe2bb66
|
| 4 |
+
|
| 5 |
+
def add_op_namespace_prefix(op_name: str):
|
| 6 |
+
"""
|
| 7 |
+
Prefix op by namespace.
|
| 8 |
+
"""
|
| 9 |
+
return f"_aiter_flash_attn_ck_rocm_fe2bb66::{op_name}"
|
build/torch210-rocm71-x86_64-linux/metadata.json
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "aiter-flash-attn-ck",
|
| 3 |
+
"id": "_aiter_flash_attn_ck_rocm_fe2bb66",
|
| 4 |
+
"version": 1,
|
| 5 |
+
"kernels-minver": "0.14.0",
|
| 6 |
+
"license": "MIT",
|
| 7 |
+
"python-depends": [],
|
| 8 |
+
"kernel-depends": [],
|
| 9 |
+
"backend": {
|
| 10 |
+
"type": "rocm",
|
| 11 |
+
"archs": [
|
| 12 |
+
"gfx942",
|
| 13 |
+
"gfx950"
|
| 14 |
+
]
|
| 15 |
+
},
|
| 16 |
+
"digest": {
|
| 17 |
+
"algorithm": "sha256",
|
| 18 |
+
"files": {
|
| 19 |
+
"__init__.py": "7wj1GTm2bqTLLCyT4xLuAk/+JWYrhINjY/ODMNWhgZ0=",
|
| 20 |
+
"_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so": "BK68IMo96IzQyxN/ujqvCqixzNFliN9mDHDISRZT8Bw=",
|
| 21 |
+
"_ops.py": "4WkdYDWWJp60zep+oW/5ym9vIeVMNfrgRCkP3o/w7yU=",
|
| 22 |
+
"mha.py": "zCAjY4O+DiCzmEL+W9sqXxZaqjsj4RJbWFcyPpUZejk=",
|
| 23 |
+
"symbols.json": "rhnF4H8Jsb6I+YPxIVFMgnLcTQJODEXbZ7ge77NZXAU="
|
| 24 |
+
}
|
| 25 |
+
},
|
| 26 |
+
"provenance": {
|
| 27 |
+
"kernel-builder": {
|
| 28 |
+
"version": "0.18.0-dev0",
|
| 29 |
+
"commit": "d054906a13710b811b96697b4635f03020842733",
|
| 30 |
+
"dirty": false
|
| 31 |
+
},
|
| 32 |
+
"kernel": {
|
| 33 |
+
"commit": "fe2bb66fee6c2a77eccf08a0508dc57428d488ed",
|
| 34 |
+
"dirty": false
|
| 35 |
+
}
|
| 36 |
+
}
|
| 37 |
+
}
|
build/torch210-rocm71-x86_64-linux/metadata.json.sigstore
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json", "verificationMaterial":{"certificate":{"rawBytes":"MIIHLDCCBrKgAwIBAgIUMp+DSFw+JgfWV7TcLB0c0Mb65MwwCgYIKoZIzj0EAwMwNzEVMBMGA1UEChMMc2lnc3RvcmUuZGV2MR4wHAYDVQQDExVzaWdzdG9yZS1pbnRlcm1lZGlhdGUwHhcNMjYxMDE5MjIzMTExWhcNMjYxMDE5MjI0MTExWjAAMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE8kLbyyRJUo+k0QdJbUOn+ybnTHhtfKEr7dpiN7ICGRBSv7puurajV7kDVk6RJbao8Okhui99gTQi7oGPT0Nxt6OCBdEwggXNMA4GA1UdDwEB/wQEAwIHgDATBgNVHSUEDDAKBggrBgEFBQcDAzAfBgNVHSMEGDAWgBTf0+nPViQRlvmo2OkoVaLGLhhkPzBrBgNVHREBAf8EYTBfhl1odHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkvLmdpdGh1Yi93b3JrZmxvd3MvYnVpbGQueWFtbEByZWZzL2hlYWRzL21haW4wOQYKKwYBBAGDvzABAQQraHR0cHM6Ly90b2tlbi5hY3Rpb25zLmdpdGh1YnVzZXJjb250ZW50LmNvbTAfBgorBgEEAYO/MAECBBF3b3JrZmxvd19kaXNwYXRjaDA2BgorBgEEAYO/MAEDBChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMBMGCisGAQQBg78wAQQEBUJ1aWxkMCsGCisGAQQBg78wAQUEHWh1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5MB0GCisGAQQBg78wAQYED3JlZnMvaGVhZHMvbWFpbjA7BgorBgEEAYO/MAEIBC0MK2h0dHBzOi8vdG9rZW4uYWN0aW9ucy5naXRodWJ1c2VyY29udGVudC5jb20wbQYKKwYBBAGDvzABCQRfDF1odHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkvLmdpdGh1Yi93b3JrZmxvd3MvYnVpbGQueWFtbEByZWZzL2hlYWRzL21haW4wOAYKKwYBBAGDvzABCgQqDChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMBsGCisGAQQBg78wAQsEDQwLc2VsZi1ob3N0ZWQwQAYKKwYBBAGDvzABDAQyDDBodHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkwOAYKKwYBBAGDvzABDQQqDChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMB8GCisGAQQBg78wAQ4EEQwPcmVmcy9oZWFkcy9tYWluMBoGCisGAQQBg78wAQ8EDAwKMTA3MTQ3NTUyOTAuBgorBgEEAYO/MAEQBCAMHmh0dHBzOi8vZ2l0aHViLmNvbS9odWdnaW5nZmFjZTAYBgorBgEEAYO/MAERBAoMCDI1NzIwNzQzMG0GCisGAQQBg78wARIEXwxdaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5Ly5naXRodWIvd29ya2Zsb3dzL2J1aWxkLnlhbWxAcmVmcy9oZWFkcy9tYWluMDgGCisGAQQBg78wARMEKgwoZmUyYmI2NmZlZTZjMmE3N2VjY2YwOGEwNTA4ZGM1NzQyOGQ0ODhlZDAhBgorBgEEAYO/MAEUBBMMEXdvcmtmbG93X2Rpc3BhdGNoMGQGCisGAQQBg78wARUEVgxUaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5L2FjdGlvbnMvcnVucy8zOTIzOTUxODEwNC9hdHRlbXB0cy8xMBYGCisGAQQBg78wARYECAwGcHVibGljMEYGCisGAQQBg78wARgEOAw2cmVwbzpodWdnaW5nZmFjZS9rZXJuZWxzLWNvbW11bml0eTpyZWY6cmVmcy9oZWFkcy9tYWluMIGKBgorBgEEAdZ5AgQCBHwEegB4AHYA3T0wasbHETJjGR4cmWc3AqJKXrjePK3/h4pygC8p7o4AAAGhVkpJ0wAABAMARzBFAiBd9Y7k3OakcDXOUOtPyvjvTHZaUz/B/tF4M/lrAPbSkgIhAMS6DTW0k6oKIEa1SATYHGWoTmXaF3t6lEktxjLrj5OMMAoGCCqGSM49BAMDA2gAMGUCMH23PJfryGMZHm69vca7K+4CeSSdtIsPRohlbI62CVTQgx/5zELDp54UGy2s+s8O0AIxAIDnrnYiN4CaOuqRlp6WpPLQuG0Xvzk3XM/ioaBbCV9mGfH1y674/krGMefcljicUw=="}, "tlogEntries":[{"logIndex":"3327211242", "logId":{"keyId":"wNI9atQGlz+VWfO6LRygH4QUfY/8W4RFwiT5i5WRgB0="}, "kindVersion":{"kind":"hashedrekord", "version":"0.0.1"}, "integratedTime":"1792449071", "inclusionPromise":{"signedEntryTimestamp":"MEQCIAr6PySJYJqy9DPHTtQgj9lSQLP+7jzp1PQhaQdDBhOkAiATFyyPYAtt+8DMi8K3HCc98mwBas4Kn6GOtrmIo+u8cA=="}, "inclusionProof":{"logIndex":"3205306980", "rootHash":"/n2pIUbsEGrPvDVhc62xJVK1pGu6X6FIjXWKAB9B11s=", "treeSize":"3205307004", "hashes":["b8JPr2VEt72nRhnrDm3LGBUw57uu9LLjYUP7BrgM9j8=", "uj8iz9+habwy51518PtXayHwhXS1UVWHasW4Dw4HEv8=", "ZzHzDD4a8QdIuk81wimU52tqU/HqEaVUd5zE4szbH5k=", "hKyjtQs1OpKphopD5PuSJ+QLeWBbsxl1l8V8vcQPgug=", "GNln8dir9ny0f06ahhipF8UWPa4XL9o9Bf1LV9qfH9M=", "v5NV1PqmPcm+8RACz+834UW43Jvx9T42P+ziCOAZqEY=", "t5D+40j8Z6l/36YGQvBik1shdJk9qiXxj5J6yRy+Ilw=", "t4Yy4kaoiJ3+VosqXw5Ogmg+A7p/E6y8+JhqWbalMbI=", "5egUlZ4Pi0Ix/qLyCDXocsWywJCHqrde/S06vOZvlOM=", "QsXufUpoKtSjKAB19K1G00umQpWvPHlBNuRGFiiVjQQ=", "QIMkwQbwAewhLJd//5FFwI3SnyNQBGjCgJrelfG8vhc=", "Et17CACUv9QN64irUJuBjsrgRGkGnnJ9xacRV/aBdwQ=", "lqgEwkxhDMs/gdIAE4lmjfMwg7nDmv7lda6O0LGliCw=", "HovTPBWzoErvUg1u5sIi3x1ZonZ7yJekVsBrmpomCIA=", "G+nk7mgE5h6dUmv2UY7NDJrY3WvrZY31raaO1AETt+o=", "qlTWNDzERnTBQVcHHXXVPjiUP0SsikGxfcxVlgqVFDo=", "gg9GNApNke3+zNhl4tsWtmIycCFrOMCB3lys6XuSQNQ=", "NlOclldmKPA1tMKIskqHAwuQDYHp31YwpK+OOMrMccc=", "qxzHanAzz57SDdmJe0B7bJK72NTIBbwEMGBBKvDOROw=", "xH/DCseLHr9eKoYT8qsORZK7zVdEGYWHuVtsVrD95wY="], "checkpoint":{"envelope":"rekor.sigstore.dev - 1193050959916656506\n3205307004\n/n2pIUbsEGrPvDVhc62xJVK1pGu6X6FIjXWKAB9B11s=\n\n— rekor.sigstore.dev wNI9ajBFAiAbIQNvzdYi+kJoZke0K/Pqgv3/lAKAHbSNYL82CNBKXQIhAJl7oEPKWHot9DzcEzfhRkqZL/pbPqMksdsMLXmuh/4s\n"}}, "canonicalizedBody":"eyJhcGlWZXJzaW9uIjoiMC4wLjEiLCJraW5kIjoiaGFzaGVkcmVrb3JkIiwic3BlYyI6eyJkYXRhIjp7Imhhc2giOnsiYWxnb3JpdGhtIjoic2hhMjU2IiwidmFsdWUiOiI5MWY4ZDQxN2NiYzEyMDY4YjIyMGQwNmY4ZmYxNTg5MzBiNTdjZTc1ZjVjMzkzOTc5MTQwOTgwNzQ1YThmNTQyIn19LCJzaWduYXR1cmUiOnsiY29udGVudCI6Ik1FUUNJR0hqWFJuSmpUbnFaaVZ1QmpNZU9oYU54aHIwSFE2enpSckI0cVBlN1Y5M0FpQm9QeS9kT1FLcVZNOWlnTGpSbmhOeFFBQ1plLzM4YWZLSFBqMExNZk5NdUE9PSIsInB1YmxpY0tleSI6eyJjb250ZW50IjoiTFMwdExTMUNSVWRKVGlCRFJWSlVTVVpKUTBGVVJTMHRMUzB0Q2sxSlNVaE1SRU5EUW5KTFowRjNTVUpCWjBsVlRYQXJSRk5HZHl0S1oyWlhWamRVWTB4Q01HTXdUV0kyTlUxM2QwTm5XVWxMYjFwSmVtb3dSVUYzVFhjS1RucEZWazFDVFVkQk1WVkZRMmhOVFdNeWJHNWpNMUoyWTIxVmRWcEhWakpOVWpSM1NFRlpSRlpSVVVSRmVGWjZZVmRrZW1SSE9YbGFVekZ3WW01U2JBcGpiVEZzV2tkc2FHUkhWWGRJYUdOT1RXcFplRTFFUlRWTmFrbDZUVlJGZUZkb1kwNU5hbGw0VFVSRk5VMXFTVEJOVkVWNFYycEJRVTFHYTNkRmQxbElDa3R2V2tsNmFqQkRRVkZaU1V0dldrbDZhakJFUVZGalJGRm5RVVU0YTB4aWVYbFNTbFZ2SzJzd1VXUktZbFZQYml0NVltNVVTR2gwWmt0RmNqZGtjR2tLVGpkSlEwZFNRbE4yTjNCMWRYSmhhbFkzYTBSV2F6WlNTbUpoYnpoUGEyaDFhVGs1WjFSUmFUZHZSMUJVTUU1NGREWlBRMEprUlhkbloxaE9UVUUwUndwQk1WVmtSSGRGUWk5M1VVVkJkMGxJWjBSQlZFSm5UbFpJVTFWRlJFUkJTMEpuWjNKQ1owVkdRbEZqUkVGNlFXWkNaMDVXU0ZOTlJVZEVRVmRuUWxSbUNqQXJibEJXYVZGU2JIWnRiekpQYTI5V1lVeEhUR2hvYTFCNlFuSkNaMDVXU0ZKRlFrRm1PRVZaVkVKbWFHd3hiMlJJVW5kamVtOTJUREprY0dSSGFERUtXV2sxYW1JeU1IWmhTRlp1V2pKc2RWb3lXbWhaTWxWMllUSldlV0p0Vm5OamVURnFZakl4ZEdSWE5YQmtTR3QyVEcxa2NHUkhhREZaYVRrellqTktjZ3BhYlhoMlpETk5kbGx1Vm5CaVIxRjFaVmRHZEdKRlFubGFWMXA2VERKb2JGbFhVbnBNTWpGb1lWYzBkMDlSV1V0TGQxbENRa0ZIUkhaNlFVSkJVVkZ5Q21GSVVqQmpTRTAyVEhrNU1HSXlkR3hpYVRWb1dUTlNjR0l5TlhwTWJXUndaRWRvTVZsdVZucGFXRXBxWWpJMU1GcFhOVEJNYlU1MllsUkJaa0puYjNJS1FtZEZSVUZaVHk5TlFVVkRRa0pHTTJJelNuSmFiWGgyWkRFNWEyRllUbmRaV0ZKcVlVUkJNa0puYjNKQ1owVkZRVmxQTDAxQlJVUkNRMmh0V2xSS2FRcFphbGt5V20xV2JFNXRUWGxaVkdNeldsZE9hbHBxUVRSWlZFRXhUVVJvYTFsNlZUTk9SRWswV2tSUk5FOUhWbXROUWsxSFEybHpSMEZSVVVKbk56aDNDa0ZSVVVWQ1ZVb3hZVmQ0YTAxRGMwZERhWE5IUVZGUlFtYzNPSGRCVVZWRlNGZG9NVm95WkhCaWJXUnRXVmRPYkV3eWRHeGpiVFZzWWtoTmRGa3lPWFFLWWxoV2RXRllValZOUWpCSFEybHpSMEZSVVVKbk56aDNRVkZaUlVRelNteGFiazEyWVVkV2FGcElUWFppVjBad1ltcEJOMEpuYjNKQ1owVkZRVmxQTHdwTlFVVkpRa013VFVzeWFEQmtTRUo2VDJrNGRtUkhPWEphVnpSMVdWZE9NR0ZYT1hWamVUVnVZVmhTYjJSWFNqRmpNbFo1V1RJNWRXUkhWblZrUXpWcUNtSXlNSGRpVVZsTFMzZFpRa0pCUjBSMmVrRkNRMUZTWmtSR01XOWtTRkozWTNwdmRrd3laSEJrUjJneFdXazFhbUl5TUhaaFNGWnVXakpzZFZveVdtZ0tXVEpWZG1FeVZubGliVlp6WTNreGFtSXlNWFJrVnpWd1pFaHJka3h0WkhCa1IyZ3hXV2s1TTJJelNuSmFiWGgyWkROTmRsbHVWbkJpUjFGMVpWZEdkQXBpUlVKNVdsZGFla3d5YUd4WlYxSjZUREl4YUdGWE5IZFBRVmxMUzNkWlFrSkJSMFIyZWtGQ1EyZFJjVVJEYUcxYVZFcHBXV3BaTWxwdFZteE9iVTE1Q2xsVVl6TmFWMDVxV21wQk5GbFVRVEZOUkdocldYcFZNMDVFU1RSYVJGRTBUMGRXYTAxQ2MwZERhWE5IUVZGUlFtYzNPSGRCVVhORlJGRjNUR015Vm5NS1dta3hiMkl6VGpCYVYxRjNVVUZaUzB0M1dVSkNRVWRFZG5wQlFrUkJVWGxFUkVKdlpFaFNkMk42YjNaTU1tUndaRWRvTVZscE5XcGlNakIyWVVoV2JncGFNbXgxV2pKYWFGa3lWWFpoTWxaNVltMVdjMk41TVdwaU1qRjBaRmMxY0dSSWEzZFBRVmxMUzNkWlFrSkJSMFIyZWtGQ1JGRlJjVVJEYUcxYVZFcHBDbGxxV1RKYWJWWnNUbTFOZVZsVVl6TmFWMDVxV21wQk5GbFVRVEZOUkdocldYcFZNMDVFU1RSYVJGRTBUMGRXYTAxQ09FZERhWE5IUVZGUlFtYzNPSGNLUVZFMFJVVlJkMUJqYlZadFkzazViMXBYUm10amVUbDBXVmRzZFUxQ2IwZERhWE5IUVZGUlFtYzNPSGRCVVRoRlJFRjNTMDFVUVROTlZGRXpUbFJWZVFwUFZFRjFRbWR2Y2tKblJVVkJXVTh2VFVGRlVVSkRRVTFJYldnd1pFaENlazlwT0haYU1td3dZVWhXYVV4dFRuWmlVemx2WkZka2JtRlhOVzVhYlVacUNscFVRVmxDWjI5eVFtZEZSVUZaVHk5TlFVVlNRa0Z2VFVORVNURk9la2wzVG5wUmVrMUhNRWREYVhOSFFWRlJRbWMzT0hkQlVrbEZXSGQ0WkdGSVVqQUtZMGhOTmt4NU9XNWhXRkp2WkZkSmRWa3lPWFJNTW1neFdqSmtjR0p0WkcxWlYwNXNUREowYkdOdE5XeGlTRTEwV1RJNWRHSllWblZoV0ZJMVRIazFiZ3BoV0ZKdlpGZEpkbVF5T1hsaE1scHpZak5rZWt3eVNqRmhWM2hyVEc1c2FHSlhlRUZqYlZadFkzazViMXBYUm10amVUbDBXVmRzZFUxRVowZERhWE5IQ2tGUlVVSm5OemgzUVZKTlJVdG5kMjlhYlZWNVdXMUpNazV0V214YVZGcHFUVzFGTTA0eVZtcFpNbGwzVDBkRmQwNVVRVFJhUjAweFRucFJlVTlIVVRBS1QwUm9iRnBFUVdoQ1oyOXlRbWRGUlVGWlR5OU5RVVZWUWtKTlRVVllaSFpqYlhSdFlrYzVNMWd5VW5Cak0wSm9aRWRPYjAxSFVVZERhWE5IUVZGUlFncG5OemgzUVZKVlJWWm5lRlZoU0ZJd1kwaE5Oa3g1T1c1aFdGSnZaRmRKZFZreU9YUk1NbWd4V2pKa2NHSnRaRzFaVjA1c1RESjBiR050Tld4aVNFMTBDbGt5T1hSaVdGWjFZVmhTTlV3eVJtcGtSMngyWW01TmRtTnVWblZqZVRoNlQxUkplazlVVlhoUFJFVjNUa001YUdSSVVteGlXRUl3WTNrNGVFMUNXVWNLUTJselIwRlJVVUpuTnpoM1FWSlpSVU5CZDBkalNGWnBZa2RzYWsxRldVZERhWE5IUVZGUlFtYzNPSGRCVW1kRlQwRjNNbU50Vm5kaWVuQnZaRmRrYmdwaFZ6VnVXbTFHYWxwVE9YSmFXRXAxV2xkNGVreFhUblppVnpFeFltMXNNR1ZVY0hsYVYxazJZMjFXYldONU9XOWFWMFpyWTNrNWRGbFhiSFZOU1VkTENrSm5iM0pDWjBWRlFXUmFOVUZuVVVOQ1NIZEZaV2RDTkVGSVdVRXpWREIzWVhOaVNFVlVTbXBIVWpSamJWZGpNMEZ4U2t0WWNtcGxVRXN6TDJnMGNIa0taME00Y0Rkdk5FRkJRVWRvVm10d1NqQjNRVUZDUVUxQlVucENSa0ZwUW1RNVdUZHJNMDloYTJORVdFOVZUM1JRZVhacWRsUklXbUZWZWk5Q0wzUkdOQXBOTDJ4eVFWQmlVMnRuU1doQlRWTTJSRlJYTUdzMmIwdEpSV0V4VTBGVVdVaEhWMjlVYlZoaFJqTjBObXhGYTNSNGFreHlhalZQVFUxQmIwZERRM0ZIQ2xOTk5EbENRVTFFUVRKblFVMUhWVU5OU0RJelVFcG1jbmxIVFZwSWJUWTVkbU5oTjBzck5FTmxVMU5rZEVselVGSnZhR3hpU1RZeVExWlVVV2Q0THpVS2VrVk1SSEExTkZWSGVUSnpLM000VHpCQlNYaEJTVVJ1Y201WmFVNDBRMkZQZFhGU2JIQTJWM0JRVEZGMVJ6QllkbnByTTFoTkwybHZZVUppUTFZNWJRcEhaa2d4ZVRZM05DOXJja2ROWldaamJHcHBZMVYzUFQwS0xTMHRMUzFGVGtRZ1EwVlNWRWxHU1VOQlZFVXRMUzB0TFFvPSJ9fX19"}], "timestampVerificationData":{"rfc3161Timestamps":[{"signedTimestamp":"MIICyzADAgEAMIICwgYJKoZIhvcNAQcCoIICszCCAq8CAQMxDTALBglghkgBZQMEAgEwgbgGCyqGSIb3DQEJEAEEoIGoBIGlMIGiAgEBBgkrBgEEAYO/MAIwMTANBglghkgBZQMEAgEFAAQghhLtVKNwP9C9AoYe4HEUs+7awj5hU/huvyNh5KwKZwoCFQDsr/tPNk6bbSMFPHHhg8PZMEWfhxgPMjAyNjEwMTkyMjMxMTFaMAMCAQGgMqQwMC4xFTATBgNVBAoTDHNpZ3N0b3JlLmRldjEVMBMGA1UEAxMMc2lnc3RvcmUtdHNhoAAxggHcMIIB2AIBATBRMDkxFTATBgNVBAoTDHNpZ3N0b3JlLmRldjEgMB4GA1UEAxMXc2lnc3RvcmUtdHNhLXNlbGZzaWduZWQCFDoTVC8MkGHuvMFDL8uKjosqI4sMMAsGCWCGSAFlAwQCAaCB/DAaBgkqhkiG9w0BCQMxDQYLKoZIhvcNAQkQAQQwHAYJKoZIhvcNAQkFMQ8XDTI2MTAxOTIyMzExMVowLwYJKoZIhvcNAQkEMSIEIAVFYYjePZj3iCjN5TwF261bkicR+j86cqwhc/Ylrv1OMIGOBgsqhkiG9w0BCRACLzF/MH0wezB5BCCF+Se8B6tiysO0Q1bBDvyBssaIP9p6uebYcNnROs0FtzBVMD2kOzA5MRUwEwYDVQQKEwxzaWdzdG9yZS5kZXYxIDAeBgNVBAMTF3NpZ3N0b3JlLXRzYS1zZWxmc2lnbmVkAhQ6E1QvDJBh7rzBQy/Lio6LKiOLDDAKBggqhkjOPQQDAgRoMGYCMQCaOXxeRZ9FR864IHhKG6YZGWsx5NQvJZPbeYBSvP+tMDPnsAAdW0y0QUPJTGc2qLQCMQCaRwSvFIurXXHVpNG8n/+B9jzewECiPpGV67deqVZWywtL/TmaVHNu8YpEgd9efms="}]}}, "messageSignature":{"messageDigest":{"algorithm":"SHA2_256", "digest":"kfjUF8vBIGiyINBvj/FYkwtXznX1w5OXkUCYB0Wo9UI="}, "signature":"MEQCIGHjXRnJjTnqZiVuBjMeOhaNxhr0HQ6zzRrB4qPe7V93AiBoPy/dOQKqVM9igLjRnhNxQACZe/38afKHPj0LMfNMuA=="}}
|
build/torch210-rocm71-x86_64-linux/mha.py
ADDED
|
@@ -0,0 +1,332 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""FlashAttention forward entry points backed by aiter's Composable-Kernel FMHA.
|
| 2 |
+
|
| 3 |
+
This is the compiled (HIP / Composable Kernel) counterpart to the Triton
|
| 4 |
+
``aiter-flash-attn`` package. Only the *forward* pass is provided (dense and
|
| 5 |
+
variable-length), including support for learnable attention sinks via
|
| 6 |
+
``window_size[2]`` / ``s_aux`` (e.g. gpt-oss). The op set ships exclusively
|
| 7 |
+
the CK ``mha_fwd`` / ``mha_varlen_fwd`` kernels, so the sink path is always
|
| 8 |
+
taken (no fall-through to a non-sink ASM kernel).
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
from typing import Optional, Tuple, Union
|
| 12 |
+
|
| 13 |
+
import torch
|
| 14 |
+
from torch import Tensor
|
| 15 |
+
|
| 16 |
+
from ._ops import ops
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def _round_multiple(x: int, m: int) -> int:
|
| 20 |
+
return (x + m - 1) // m * m
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def _maybe_pad_headdim(t: Tensor, target: int) -> Tensor:
|
| 24 |
+
og = t.size(-1)
|
| 25 |
+
if og == target:
|
| 26 |
+
return t
|
| 27 |
+
return torch.nn.functional.pad(t, [0, target - og])
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def _unpack_window(window_size) -> Tuple[int, int, int]:
|
| 31 |
+
left = int(window_size[0])
|
| 32 |
+
right = int(window_size[1])
|
| 33 |
+
sink = int(window_size[2]) if len(window_size) == 3 else 0
|
| 34 |
+
return left, right, sink
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def _prep_sink(s_aux: Optional[Tensor], nheads: int, device) -> Optional[Tensor]:
|
| 38 |
+
if s_aux is None:
|
| 39 |
+
return None
|
| 40 |
+
assert s_aux.device == device, "s_aux must be on the same device as q"
|
| 41 |
+
assert s_aux.shape[0] == nheads, "s_aux must have shape [nheads_q]"
|
| 42 |
+
if s_aux.dtype != torch.float32:
|
| 43 |
+
s_aux = s_aux.to(torch.float32)
|
| 44 |
+
return s_aux
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def _effective_sink_size(sink_size: int, s_aux: Optional[Tensor]) -> int:
|
| 48 |
+
# The CK kernel only takes the sink path when sink_size > 0. Callers using
|
| 49 |
+
# the transformers attention interface pass the per-head sink logits as
|
| 50 |
+
# ``s_aux`` but leave ``window_size`` a 2-tuple (sink_size == 0), so enable
|
| 51 |
+
# a single learnable sink automatically when s_aux is given.
|
| 52 |
+
if s_aux is not None and sink_size == 0:
|
| 53 |
+
return 1
|
| 54 |
+
return sink_size
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def flash_attn_func(
|
| 58 |
+
q: Tensor,
|
| 59 |
+
k: Tensor,
|
| 60 |
+
v: Tensor,
|
| 61 |
+
dropout_p: float = 0.0,
|
| 62 |
+
softmax_scale: Optional[float] = None,
|
| 63 |
+
causal: bool = False,
|
| 64 |
+
window_size: Tuple[int, ...] = (-1, -1, 0),
|
| 65 |
+
bias: Optional[Tensor] = None,
|
| 66 |
+
alibi_slopes: Optional[Tensor] = None,
|
| 67 |
+
return_lse: bool = False,
|
| 68 |
+
return_attn_probs: bool = False,
|
| 69 |
+
s_aux: Optional[Tensor] = None,
|
| 70 |
+
**kwargs,
|
| 71 |
+
) -> Union[Tensor, Tuple[Tensor, ...]]:
|
| 72 |
+
"""Dense FlashAttention forward.
|
| 73 |
+
|
| 74 |
+
``q, k, v`` have shape ``(batch, seqlen, nheads, headdim)``. ``window_size``
|
| 75 |
+
is ``(left, right, sink_size)``; ``s_aux`` is an optional ``(nheads_q,)``
|
| 76 |
+
fp32 tensor of learnable sink logits. Extra keyword arguments (e.g. those
|
| 77 |
+
passed by the transformers flash-attention interface) are ignored.
|
| 78 |
+
"""
|
| 79 |
+
if softmax_scale is None:
|
| 80 |
+
softmax_scale = q.shape[-1] ** (-0.5)
|
| 81 |
+
window_size_left, window_size_right, sink_size = _unpack_window(window_size)
|
| 82 |
+
|
| 83 |
+
nheads_q = q.size(2)
|
| 84 |
+
s_aux = _prep_sink(s_aux, nheads_q, q.device)
|
| 85 |
+
sink_size = _effective_sink_size(sink_size, s_aux)
|
| 86 |
+
|
| 87 |
+
head_q_og = q.size(-1)
|
| 88 |
+
head_v_og = v.size(-1)
|
| 89 |
+
head_q = _round_multiple(head_q_og, 8)
|
| 90 |
+
head_v = _round_multiple(head_v_og, 8)
|
| 91 |
+
q = _maybe_pad_headdim(q, head_q)
|
| 92 |
+
k = _maybe_pad_headdim(k, head_q)
|
| 93 |
+
v = _maybe_pad_headdim(v, head_v)
|
| 94 |
+
|
| 95 |
+
return_softmax = return_attn_probs and dropout_p > 0
|
| 96 |
+
out, softmax_lse, S_dmask, _ = ops.mha_fwd(
|
| 97 |
+
q,
|
| 98 |
+
k,
|
| 99 |
+
v,
|
| 100 |
+
dropout_p,
|
| 101 |
+
softmax_scale,
|
| 102 |
+
causal,
|
| 103 |
+
window_size_left,
|
| 104 |
+
window_size_right,
|
| 105 |
+
sink_size,
|
| 106 |
+
return_lse or return_attn_probs,
|
| 107 |
+
return_softmax,
|
| 108 |
+
None, # cu_seqlens_q
|
| 109 |
+
None, # cu_seqlens_kv
|
| 110 |
+
None, # out
|
| 111 |
+
bias,
|
| 112 |
+
alibi_slopes,
|
| 113 |
+
None, # q_descale
|
| 114 |
+
None, # k_descale
|
| 115 |
+
None, # v_descale
|
| 116 |
+
s_aux,
|
| 117 |
+
None, # gen
|
| 118 |
+
)
|
| 119 |
+
|
| 120 |
+
out = out[..., :head_v_og]
|
| 121 |
+
if not (return_lse or return_attn_probs):
|
| 122 |
+
return out
|
| 123 |
+
result = (out,)
|
| 124 |
+
if return_lse:
|
| 125 |
+
result = result + (softmax_lse,)
|
| 126 |
+
if return_attn_probs:
|
| 127 |
+
result = result + (S_dmask,)
|
| 128 |
+
return result
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
def flash_attn_varlen_func(
|
| 132 |
+
q: Tensor,
|
| 133 |
+
k: Tensor,
|
| 134 |
+
v: Tensor,
|
| 135 |
+
cu_seqlens_q: Tensor,
|
| 136 |
+
cu_seqlens_k: Tensor,
|
| 137 |
+
max_seqlen_q: int,
|
| 138 |
+
max_seqlen_k: int,
|
| 139 |
+
dropout_p: float = 0.0,
|
| 140 |
+
softmax_scale: Optional[float] = None,
|
| 141 |
+
causal: bool = False,
|
| 142 |
+
window_size: Tuple[int, ...] = (-1, -1, 0),
|
| 143 |
+
bias: Optional[Tensor] = None,
|
| 144 |
+
alibi_slopes: Optional[Tensor] = None,
|
| 145 |
+
return_lse: bool = False,
|
| 146 |
+
return_attn_probs: bool = False,
|
| 147 |
+
block_table: Optional[Tensor] = None,
|
| 148 |
+
min_seqlen_q: int = 0,
|
| 149 |
+
logits_soft_cap: float = 0.0,
|
| 150 |
+
zero_tensors: bool = False,
|
| 151 |
+
cu_seqlens_q_padded: Optional[Tensor] = None,
|
| 152 |
+
cu_seqlens_k_padded: Optional[Tensor] = None,
|
| 153 |
+
s_aux: Optional[Tensor] = None,
|
| 154 |
+
**kwargs,
|
| 155 |
+
) -> Union[Tensor, Tuple[Tensor, ...]]:
|
| 156 |
+
"""Variable-length / packed FlashAttention forward.
|
| 157 |
+
|
| 158 |
+
``q`` has shape ``(total_q, nheads, headdim)``; ``cu_seqlens_*`` are int32
|
| 159 |
+
cumulative offsets of shape ``(batch + 1,)``. Sinks are supported through
|
| 160 |
+
``window_size[2]`` / ``s_aux`` as in :func:`flash_attn_func`. Extra keyword
|
| 161 |
+
arguments (e.g. from the transformers flash-attention interface) are ignored.
|
| 162 |
+
"""
|
| 163 |
+
if softmax_scale is None:
|
| 164 |
+
softmax_scale = q.shape[-1] ** (-0.5)
|
| 165 |
+
window_size_left, window_size_right, sink_size = _unpack_window(window_size)
|
| 166 |
+
|
| 167 |
+
nheads_q = q.size(1)
|
| 168 |
+
s_aux = _prep_sink(s_aux, nheads_q, q.device)
|
| 169 |
+
sink_size = _effective_sink_size(sink_size, s_aux)
|
| 170 |
+
|
| 171 |
+
head_q_og = q.size(-1)
|
| 172 |
+
head_v_og = v.size(-1)
|
| 173 |
+
head_q = _round_multiple(head_q_og, 8)
|
| 174 |
+
head_v = _round_multiple(head_v_og, 8)
|
| 175 |
+
q = _maybe_pad_headdim(q, head_q)
|
| 176 |
+
k = _maybe_pad_headdim(k, head_q)
|
| 177 |
+
v = _maybe_pad_headdim(v, head_v)
|
| 178 |
+
|
| 179 |
+
return_softmax = return_attn_probs and dropout_p > 0
|
| 180 |
+
out, softmax_lse, S_dmask, _ = ops.mha_varlen_fwd(
|
| 181 |
+
q,
|
| 182 |
+
k,
|
| 183 |
+
v,
|
| 184 |
+
cu_seqlens_q,
|
| 185 |
+
cu_seqlens_k,
|
| 186 |
+
max_seqlen_q,
|
| 187 |
+
max_seqlen_k,
|
| 188 |
+
min_seqlen_q,
|
| 189 |
+
dropout_p,
|
| 190 |
+
softmax_scale,
|
| 191 |
+
logits_soft_cap,
|
| 192 |
+
zero_tensors,
|
| 193 |
+
causal,
|
| 194 |
+
window_size_left,
|
| 195 |
+
window_size_right,
|
| 196 |
+
sink_size,
|
| 197 |
+
return_lse or return_attn_probs,
|
| 198 |
+
return_softmax,
|
| 199 |
+
None, # out
|
| 200 |
+
block_table,
|
| 201 |
+
bias,
|
| 202 |
+
alibi_slopes,
|
| 203 |
+
None, # q_descale
|
| 204 |
+
None, # k_descale
|
| 205 |
+
None, # v_descale
|
| 206 |
+
None, # gen
|
| 207 |
+
cu_seqlens_q_padded,
|
| 208 |
+
cu_seqlens_k_padded,
|
| 209 |
+
s_aux,
|
| 210 |
+
)
|
| 211 |
+
|
| 212 |
+
out = out[..., :head_v_og]
|
| 213 |
+
if not (return_lse or return_attn_probs):
|
| 214 |
+
return out
|
| 215 |
+
result = (out,)
|
| 216 |
+
if return_lse:
|
| 217 |
+
result = result + (softmax_lse,)
|
| 218 |
+
if return_attn_probs:
|
| 219 |
+
result = result + (S_dmask,)
|
| 220 |
+
return result
|
| 221 |
+
|
| 222 |
+
|
| 223 |
+
def _append_to_contiguous_cache(k_cache, v_cache, k, v, cache_seqlens, Sq):
|
| 224 |
+
"""In-place append of new k/v into a contiguous cache at per-batch offsets."""
|
| 225 |
+
B = k_cache.shape[0]
|
| 226 |
+
if isinstance(cache_seqlens, int):
|
| 227 |
+
cache_seqlens = torch.full((B,), cache_seqlens, device=k_cache.device, dtype=torch.long)
|
| 228 |
+
pos = cache_seqlens.view(B, 1).to(torch.long) + torch.arange(Sq, device=k_cache.device).view(1, Sq)
|
| 229 |
+
bidx = torch.arange(B, device=k_cache.device).view(B, 1).expand(B, Sq)
|
| 230 |
+
k_cache[bidx, pos] = k.to(k_cache.dtype)
|
| 231 |
+
v_cache[bidx, pos] = v.to(v_cache.dtype)
|
| 232 |
+
|
| 233 |
+
|
| 234 |
+
def flash_attn_with_kvcache(
|
| 235 |
+
q: Tensor,
|
| 236 |
+
k_cache: Tensor,
|
| 237 |
+
v_cache: Tensor,
|
| 238 |
+
k: Optional[Tensor] = None,
|
| 239 |
+
v: Optional[Tensor] = None,
|
| 240 |
+
rotary_cos: Optional[Tensor] = None,
|
| 241 |
+
rotary_sin: Optional[Tensor] = None,
|
| 242 |
+
cache_seqlens=None,
|
| 243 |
+
block_table: Optional[Tensor] = None,
|
| 244 |
+
softmax_scale: Optional[float] = None,
|
| 245 |
+
causal: bool = True,
|
| 246 |
+
window_size: Tuple[int, ...] = (-1, -1, 0),
|
| 247 |
+
s_aux: Optional[Tensor] = None,
|
| 248 |
+
return_lse: bool = False,
|
| 249 |
+
**kwargs,
|
| 250 |
+
) -> Union[Tensor, Tuple[Tensor, ...]]:
|
| 251 |
+
"""FlashAttention against a KV cache, built on the CK split-KV kernel.
|
| 252 |
+
|
| 253 |
+
``q`` is ``(batch, seqlen_q, nheads, headdim)``. For a contiguous cache,
|
| 254 |
+
``k_cache``/``v_cache`` are ``(batch, seqlen_cache, nheads_k, headdim)`` and
|
| 255 |
+
``cache_seqlens`` (int or ``(batch,)``) is the number of valid cache entries
|
| 256 |
+
*before* appending. If ``k``/``v`` (the new tokens) are given, they are
|
| 257 |
+
written into the cache in place. The query attends to the valid prefix
|
| 258 |
+
(``cache_seqlens`` + ``seqlen_q``) with bottom-right causal alignment.
|
| 259 |
+
|
| 260 |
+
Notes / current limitations:
|
| 261 |
+
- Rotary embedding is not applied here; pass ``q``/``k`` already rotated
|
| 262 |
+
(``rotary_cos``/``rotary_sin`` are rejected).
|
| 263 |
+
- Paged caches are addressed via ``block_table`` (forwarded to the split-KV
|
| 264 |
+
varlen kernel); in-cache append for paged layouts must be done by the
|
| 265 |
+
caller (pass ``k=v=None``).
|
| 266 |
+
"""
|
| 267 |
+
if rotary_cos is not None or rotary_sin is not None:
|
| 268 |
+
raise NotImplementedError(
|
| 269 |
+
"flash_attn_with_kvcache does not apply rotary; rotate q/k beforehand."
|
| 270 |
+
)
|
| 271 |
+
if softmax_scale is None:
|
| 272 |
+
softmax_scale = q.shape[-1] ** (-0.5)
|
| 273 |
+
window_size_left, window_size_right, sink_size = _unpack_window(window_size)
|
| 274 |
+
B, Sq, Hq, D = q.shape
|
| 275 |
+
s_aux = _prep_sink(s_aux, Hq, q.device)
|
| 276 |
+
sink_size = _effective_sink_size(sink_size, s_aux)
|
| 277 |
+
|
| 278 |
+
paged = block_table is not None
|
| 279 |
+
|
| 280 |
+
# ---- append new tokens into the cache (contiguous layout only) ----
|
| 281 |
+
if k is not None and not paged:
|
| 282 |
+
assert cache_seqlens is not None, "cache_seqlens required to append into the cache"
|
| 283 |
+
_append_to_contiguous_cache(k_cache, v_cache, k, v, cache_seqlens, Sq)
|
| 284 |
+
elif k is not None and paged:
|
| 285 |
+
raise NotImplementedError(
|
| 286 |
+
"in-cache append for paged layouts is not supported; append externally "
|
| 287 |
+
"and call with k=v=None."
|
| 288 |
+
)
|
| 289 |
+
|
| 290 |
+
appended = 0 if k is None else Sq
|
| 291 |
+
|
| 292 |
+
# ---- key lengths ----
|
| 293 |
+
if cache_seqlens is None:
|
| 294 |
+
klens = torch.full((B,), k_cache.shape[1], device=q.device, dtype=torch.long)
|
| 295 |
+
elif isinstance(cache_seqlens, int):
|
| 296 |
+
klens = torch.full((B,), cache_seqlens + appended, device=q.device, dtype=torch.long)
|
| 297 |
+
else:
|
| 298 |
+
klens = cache_seqlens.to(q.device, torch.long) + appended
|
| 299 |
+
|
| 300 |
+
uniform = bool((klens == klens[0]).all().item())
|
| 301 |
+
|
| 302 |
+
# ---- fast path: contiguous cache, uniform length -> dense kernel ----
|
| 303 |
+
if not paged and uniform:
|
| 304 |
+
L = int(klens[0].item())
|
| 305 |
+
return flash_attn_func(
|
| 306 |
+
q, k_cache[:, :L], v_cache[:, :L],
|
| 307 |
+
softmax_scale=softmax_scale, causal=causal,
|
| 308 |
+
window_size=(window_size_left, window_size_right, sink_size),
|
| 309 |
+
s_aux=s_aux, return_lse=return_lse,
|
| 310 |
+
)
|
| 311 |
+
|
| 312 |
+
# ---- general path: variable cache lengths -> split-KV varlen kernel ----
|
| 313 |
+
cu_q = torch.arange(0, (B + 1) * Sq, Sq, device=q.device, dtype=torch.int32)
|
| 314 |
+
cu_k = torch.zeros(B + 1, device=q.device, dtype=torch.int32)
|
| 315 |
+
cu_k[1:] = torch.cumsum(klens, 0).to(torch.int32)
|
| 316 |
+
q_packed = q.reshape(B * Sq, Hq, D)
|
| 317 |
+
if paged:
|
| 318 |
+
k_packed, v_packed = k_cache, v_cache # kernel gathers via block_table
|
| 319 |
+
else:
|
| 320 |
+
k_packed = torch.cat([k_cache[b, : int(klens[b])] for b in range(B)], dim=0)
|
| 321 |
+
v_packed = torch.cat([v_cache[b, : int(klens[b])] for b in range(B)], dim=0)
|
| 322 |
+
|
| 323 |
+
out = flash_attn_varlen_func(
|
| 324 |
+
q_packed, k_packed, v_packed, cu_q, cu_k, Sq, int(klens.max().item()),
|
| 325 |
+
softmax_scale=softmax_scale, causal=causal,
|
| 326 |
+
window_size=(window_size_left, window_size_right, sink_size),
|
| 327 |
+
block_table=block_table, s_aux=s_aux, return_lse=return_lse,
|
| 328 |
+
)
|
| 329 |
+
if return_lse:
|
| 330 |
+
out, lse = out
|
| 331 |
+
return out.reshape(B, Sq, Hq, D), lse
|
| 332 |
+
return out.reshape(B, Sq, Hq, D)
|
build/torch210-rocm71-x86_64-linux/symbols.json
ADDED
|
@@ -0,0 +1,349 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": 1,
|
| 3 |
+
"module": "_aiter_flash_attn_ck_rocm_fe2bb66",
|
| 4 |
+
"functions": [
|
| 5 |
+
{
|
| 6 |
+
"name": "flash_attn_func",
|
| 7 |
+
"qualname": "flash_attn_func",
|
| 8 |
+
"module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha",
|
| 9 |
+
"kind": "function",
|
| 10 |
+
"doc": "Dense FlashAttention forward.\n\n``q, k, v`` have shape ``(batch, seqlen, nheads, headdim)``. ``window_size``\nis ``(left, right, sink_size)``; ``s_aux`` is an optional ``(nheads_q,)``\nfp32 tensor of learnable sink logits. Extra keyword arguments (e.g. those\npassed by the transformers flash-attention interface) are ignored.",
|
| 11 |
+
"signature": {
|
| 12 |
+
"parameters": [
|
| 13 |
+
{
|
| 14 |
+
"name": "q",
|
| 15 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 16 |
+
"default": null,
|
| 17 |
+
"annotation": "torch.Tensor"
|
| 18 |
+
},
|
| 19 |
+
{
|
| 20 |
+
"name": "k",
|
| 21 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 22 |
+
"default": null,
|
| 23 |
+
"annotation": "torch.Tensor"
|
| 24 |
+
},
|
| 25 |
+
{
|
| 26 |
+
"name": "v",
|
| 27 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 28 |
+
"default": null,
|
| 29 |
+
"annotation": "torch.Tensor"
|
| 30 |
+
},
|
| 31 |
+
{
|
| 32 |
+
"name": "dropout_p",
|
| 33 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 34 |
+
"default": "0.0",
|
| 35 |
+
"annotation": "float"
|
| 36 |
+
},
|
| 37 |
+
{
|
| 38 |
+
"name": "softmax_scale",
|
| 39 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 40 |
+
"default": "None",
|
| 41 |
+
"annotation": "float | None"
|
| 42 |
+
},
|
| 43 |
+
{
|
| 44 |
+
"name": "causal",
|
| 45 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 46 |
+
"default": "False",
|
| 47 |
+
"annotation": "bool"
|
| 48 |
+
},
|
| 49 |
+
{
|
| 50 |
+
"name": "window_size",
|
| 51 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 52 |
+
"default": "(-1, -1, 0)",
|
| 53 |
+
"annotation": "Tuple[int, ...]"
|
| 54 |
+
},
|
| 55 |
+
{
|
| 56 |
+
"name": "bias",
|
| 57 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 58 |
+
"default": "None",
|
| 59 |
+
"annotation": "torch.Tensor | None"
|
| 60 |
+
},
|
| 61 |
+
{
|
| 62 |
+
"name": "alibi_slopes",
|
| 63 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 64 |
+
"default": "None",
|
| 65 |
+
"annotation": "torch.Tensor | None"
|
| 66 |
+
},
|
| 67 |
+
{
|
| 68 |
+
"name": "return_lse",
|
| 69 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 70 |
+
"default": "False",
|
| 71 |
+
"annotation": "bool"
|
| 72 |
+
},
|
| 73 |
+
{
|
| 74 |
+
"name": "return_attn_probs",
|
| 75 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 76 |
+
"default": "False",
|
| 77 |
+
"annotation": "bool"
|
| 78 |
+
},
|
| 79 |
+
{
|
| 80 |
+
"name": "s_aux",
|
| 81 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 82 |
+
"default": "None",
|
| 83 |
+
"annotation": "torch.Tensor | None"
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"name": "kwargs",
|
| 87 |
+
"kind": "VAR_KEYWORD",
|
| 88 |
+
"default": null,
|
| 89 |
+
"annotation": null
|
| 90 |
+
}
|
| 91 |
+
],
|
| 92 |
+
"return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]"
|
| 93 |
+
}
|
| 94 |
+
},
|
| 95 |
+
{
|
| 96 |
+
"name": "flash_attn_varlen_func",
|
| 97 |
+
"qualname": "flash_attn_varlen_func",
|
| 98 |
+
"module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha",
|
| 99 |
+
"kind": "function",
|
| 100 |
+
"doc": "Variable-length / packed FlashAttention forward.\n\n``q`` has shape ``(total_q, nheads, headdim)``; ``cu_seqlens_*`` are int32\ncumulative offsets of shape ``(batch + 1,)``. Sinks are supported through\n``window_size[2]`` / ``s_aux`` as in :func:`flash_attn_func`. Extra keyword\narguments (e.g. from the transformers flash-attention interface) are ignored.",
|
| 101 |
+
"signature": {
|
| 102 |
+
"parameters": [
|
| 103 |
+
{
|
| 104 |
+
"name": "q",
|
| 105 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 106 |
+
"default": null,
|
| 107 |
+
"annotation": "torch.Tensor"
|
| 108 |
+
},
|
| 109 |
+
{
|
| 110 |
+
"name": "k",
|
| 111 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 112 |
+
"default": null,
|
| 113 |
+
"annotation": "torch.Tensor"
|
| 114 |
+
},
|
| 115 |
+
{
|
| 116 |
+
"name": "v",
|
| 117 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 118 |
+
"default": null,
|
| 119 |
+
"annotation": "torch.Tensor"
|
| 120 |
+
},
|
| 121 |
+
{
|
| 122 |
+
"name": "cu_seqlens_q",
|
| 123 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 124 |
+
"default": null,
|
| 125 |
+
"annotation": "torch.Tensor"
|
| 126 |
+
},
|
| 127 |
+
{
|
| 128 |
+
"name": "cu_seqlens_k",
|
| 129 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 130 |
+
"default": null,
|
| 131 |
+
"annotation": "torch.Tensor"
|
| 132 |
+
},
|
| 133 |
+
{
|
| 134 |
+
"name": "max_seqlen_q",
|
| 135 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 136 |
+
"default": null,
|
| 137 |
+
"annotation": "int"
|
| 138 |
+
},
|
| 139 |
+
{
|
| 140 |
+
"name": "max_seqlen_k",
|
| 141 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 142 |
+
"default": null,
|
| 143 |
+
"annotation": "int"
|
| 144 |
+
},
|
| 145 |
+
{
|
| 146 |
+
"name": "dropout_p",
|
| 147 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 148 |
+
"default": "0.0",
|
| 149 |
+
"annotation": "float"
|
| 150 |
+
},
|
| 151 |
+
{
|
| 152 |
+
"name": "softmax_scale",
|
| 153 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 154 |
+
"default": "None",
|
| 155 |
+
"annotation": "float | None"
|
| 156 |
+
},
|
| 157 |
+
{
|
| 158 |
+
"name": "causal",
|
| 159 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 160 |
+
"default": "False",
|
| 161 |
+
"annotation": "bool"
|
| 162 |
+
},
|
| 163 |
+
{
|
| 164 |
+
"name": "window_size",
|
| 165 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 166 |
+
"default": "(-1, -1, 0)",
|
| 167 |
+
"annotation": "Tuple[int, ...]"
|
| 168 |
+
},
|
| 169 |
+
{
|
| 170 |
+
"name": "bias",
|
| 171 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 172 |
+
"default": "None",
|
| 173 |
+
"annotation": "torch.Tensor | None"
|
| 174 |
+
},
|
| 175 |
+
{
|
| 176 |
+
"name": "alibi_slopes",
|
| 177 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 178 |
+
"default": "None",
|
| 179 |
+
"annotation": "torch.Tensor | None"
|
| 180 |
+
},
|
| 181 |
+
{
|
| 182 |
+
"name": "return_lse",
|
| 183 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 184 |
+
"default": "False",
|
| 185 |
+
"annotation": "bool"
|
| 186 |
+
},
|
| 187 |
+
{
|
| 188 |
+
"name": "return_attn_probs",
|
| 189 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 190 |
+
"default": "False",
|
| 191 |
+
"annotation": "bool"
|
| 192 |
+
},
|
| 193 |
+
{
|
| 194 |
+
"name": "block_table",
|
| 195 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 196 |
+
"default": "None",
|
| 197 |
+
"annotation": "torch.Tensor | None"
|
| 198 |
+
},
|
| 199 |
+
{
|
| 200 |
+
"name": "min_seqlen_q",
|
| 201 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 202 |
+
"default": "0",
|
| 203 |
+
"annotation": "int"
|
| 204 |
+
},
|
| 205 |
+
{
|
| 206 |
+
"name": "logits_soft_cap",
|
| 207 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 208 |
+
"default": "0.0",
|
| 209 |
+
"annotation": "float"
|
| 210 |
+
},
|
| 211 |
+
{
|
| 212 |
+
"name": "zero_tensors",
|
| 213 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 214 |
+
"default": "False",
|
| 215 |
+
"annotation": "bool"
|
| 216 |
+
},
|
| 217 |
+
{
|
| 218 |
+
"name": "cu_seqlens_q_padded",
|
| 219 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 220 |
+
"default": "None",
|
| 221 |
+
"annotation": "torch.Tensor | None"
|
| 222 |
+
},
|
| 223 |
+
{
|
| 224 |
+
"name": "cu_seqlens_k_padded",
|
| 225 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 226 |
+
"default": "None",
|
| 227 |
+
"annotation": "torch.Tensor | None"
|
| 228 |
+
},
|
| 229 |
+
{
|
| 230 |
+
"name": "s_aux",
|
| 231 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 232 |
+
"default": "None",
|
| 233 |
+
"annotation": "torch.Tensor | None"
|
| 234 |
+
},
|
| 235 |
+
{
|
| 236 |
+
"name": "kwargs",
|
| 237 |
+
"kind": "VAR_KEYWORD",
|
| 238 |
+
"default": null,
|
| 239 |
+
"annotation": null
|
| 240 |
+
}
|
| 241 |
+
],
|
| 242 |
+
"return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]"
|
| 243 |
+
}
|
| 244 |
+
},
|
| 245 |
+
{
|
| 246 |
+
"name": "flash_attn_with_kvcache",
|
| 247 |
+
"qualname": "flash_attn_with_kvcache",
|
| 248 |
+
"module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha",
|
| 249 |
+
"kind": "function",
|
| 250 |
+
"doc": "FlashAttention against a KV cache, built on the CK split-KV kernel.\n\n``q`` is ``(batch, seqlen_q, nheads, headdim)``. For a contiguous cache,\n``k_cache``/``v_cache`` are ``(batch, seqlen_cache, nheads_k, headdim)`` and\n``cache_seqlens`` (int or ``(batch,)``) is the number of valid cache entries\n*before* appending. If ``k``/``v`` (the new tokens) are given, they are\nwritten into the cache in place. The query attends to the valid prefix\n(``cache_seqlens`` + ``seqlen_q``) with bottom-right causal alignment.\n\nNotes / current limitations:\n- Rotary embedding is not applied here; pass ``q``/``k`` already rotated\n (``rotary_cos``/``rotary_sin`` are rejected).\n- Paged caches are addressed via ``block_table`` (forwarded to the split-KV\n varlen kernel); in-cache append for paged layouts must be done by the\n caller (pass ``k=v=None``).",
|
| 251 |
+
"signature": {
|
| 252 |
+
"parameters": [
|
| 253 |
+
{
|
| 254 |
+
"name": "q",
|
| 255 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 256 |
+
"default": null,
|
| 257 |
+
"annotation": "torch.Tensor"
|
| 258 |
+
},
|
| 259 |
+
{
|
| 260 |
+
"name": "k_cache",
|
| 261 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 262 |
+
"default": null,
|
| 263 |
+
"annotation": "torch.Tensor"
|
| 264 |
+
},
|
| 265 |
+
{
|
| 266 |
+
"name": "v_cache",
|
| 267 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 268 |
+
"default": null,
|
| 269 |
+
"annotation": "torch.Tensor"
|
| 270 |
+
},
|
| 271 |
+
{
|
| 272 |
+
"name": "k",
|
| 273 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 274 |
+
"default": "None",
|
| 275 |
+
"annotation": "torch.Tensor | None"
|
| 276 |
+
},
|
| 277 |
+
{
|
| 278 |
+
"name": "v",
|
| 279 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 280 |
+
"default": "None",
|
| 281 |
+
"annotation": "torch.Tensor | None"
|
| 282 |
+
},
|
| 283 |
+
{
|
| 284 |
+
"name": "rotary_cos",
|
| 285 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 286 |
+
"default": "None",
|
| 287 |
+
"annotation": "torch.Tensor | None"
|
| 288 |
+
},
|
| 289 |
+
{
|
| 290 |
+
"name": "rotary_sin",
|
| 291 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 292 |
+
"default": "None",
|
| 293 |
+
"annotation": "torch.Tensor | None"
|
| 294 |
+
},
|
| 295 |
+
{
|
| 296 |
+
"name": "cache_seqlens",
|
| 297 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 298 |
+
"default": "None",
|
| 299 |
+
"annotation": null
|
| 300 |
+
},
|
| 301 |
+
{
|
| 302 |
+
"name": "block_table",
|
| 303 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 304 |
+
"default": "None",
|
| 305 |
+
"annotation": "torch.Tensor | None"
|
| 306 |
+
},
|
| 307 |
+
{
|
| 308 |
+
"name": "softmax_scale",
|
| 309 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 310 |
+
"default": "None",
|
| 311 |
+
"annotation": "float | None"
|
| 312 |
+
},
|
| 313 |
+
{
|
| 314 |
+
"name": "causal",
|
| 315 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 316 |
+
"default": "True",
|
| 317 |
+
"annotation": "bool"
|
| 318 |
+
},
|
| 319 |
+
{
|
| 320 |
+
"name": "window_size",
|
| 321 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 322 |
+
"default": "(-1, -1, 0)",
|
| 323 |
+
"annotation": "Tuple[int, ...]"
|
| 324 |
+
},
|
| 325 |
+
{
|
| 326 |
+
"name": "s_aux",
|
| 327 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 328 |
+
"default": "None",
|
| 329 |
+
"annotation": "torch.Tensor | None"
|
| 330 |
+
},
|
| 331 |
+
{
|
| 332 |
+
"name": "return_lse",
|
| 333 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 334 |
+
"default": "False",
|
| 335 |
+
"annotation": "bool"
|
| 336 |
+
},
|
| 337 |
+
{
|
| 338 |
+
"name": "kwargs",
|
| 339 |
+
"kind": "VAR_KEYWORD",
|
| 340 |
+
"default": null,
|
| 341 |
+
"annotation": null
|
| 342 |
+
}
|
| 343 |
+
],
|
| 344 |
+
"return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]"
|
| 345 |
+
}
|
| 346 |
+
}
|
| 347 |
+
],
|
| 348 |
+
"layers": []
|
| 349 |
+
}
|
build/torch211-rocm71-x86_64-linux/__init__.py
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""AITER Composable-Kernel FlashAttention kernels for AMD ROCm (compiled HIP).
|
| 2 |
+
|
| 3 |
+
Compiled (Composable Kernel / HIP) counterpart to the Triton-based
|
| 4 |
+
``aiter-flash-attn`` package, repackaged from the CK FMHA forward path of the
|
| 5 |
+
`ROCm/aiter <https://github.com/ROCm/aiter>`_ project as a self-contained
|
| 6 |
+
Hugging Face Hub kernel. Exposes the FlashAttention forward entry points with
|
| 7 |
+
learnable attention sink support (e.g. gpt-oss).
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
from .mha import (
|
| 11 |
+
flash_attn_func,
|
| 12 |
+
flash_attn_varlen_func,
|
| 13 |
+
flash_attn_with_kvcache,
|
| 14 |
+
)
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
__kernel_metadata__ = {
|
| 18 |
+
"license": "mit",
|
| 19 |
+
}
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
__all__ = [
|
| 23 |
+
"__kernel_metadata__",
|
| 24 |
+
"flash_attn_func",
|
| 25 |
+
"flash_attn_varlen_func",
|
| 26 |
+
"flash_attn_with_kvcache",
|
| 27 |
+
]
|
build/torch211-rocm71-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:8a386f0a7da580b6efa0dd9ffac2d1a4be1382a709f490ab786bf184b7fb4744
|
| 3 |
+
size 204251704
|
build/torch211-rocm71-x86_64-linux/_ops.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
from . import _aiter_flash_attn_ck_rocm_fe2bb66
|
| 3 |
+
ops = torch.ops._aiter_flash_attn_ck_rocm_fe2bb66
|
| 4 |
+
|
| 5 |
+
def add_op_namespace_prefix(op_name: str):
|
| 6 |
+
"""
|
| 7 |
+
Prefix op by namespace.
|
| 8 |
+
"""
|
| 9 |
+
return f"_aiter_flash_attn_ck_rocm_fe2bb66::{op_name}"
|
build/torch211-rocm71-x86_64-linux/metadata.json
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "aiter-flash-attn-ck",
|
| 3 |
+
"id": "_aiter_flash_attn_ck_rocm_fe2bb66",
|
| 4 |
+
"version": 1,
|
| 5 |
+
"kernels-minver": "0.14.0",
|
| 6 |
+
"license": "MIT",
|
| 7 |
+
"python-depends": [],
|
| 8 |
+
"kernel-depends": [],
|
| 9 |
+
"backend": {
|
| 10 |
+
"type": "rocm",
|
| 11 |
+
"archs": [
|
| 12 |
+
"gfx942",
|
| 13 |
+
"gfx950"
|
| 14 |
+
]
|
| 15 |
+
},
|
| 16 |
+
"digest": {
|
| 17 |
+
"algorithm": "sha256",
|
| 18 |
+
"files": {
|
| 19 |
+
"__init__.py": "7wj1GTm2bqTLLCyT4xLuAk/+JWYrhINjY/ODMNWhgZ0=",
|
| 20 |
+
"_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so": "ijhvCn2lgLbvoN2f+sLRpL4TgqcJ9JCreGvxhLf7R0Q=",
|
| 21 |
+
"_ops.py": "4WkdYDWWJp60zep+oW/5ym9vIeVMNfrgRCkP3o/w7yU=",
|
| 22 |
+
"mha.py": "zCAjY4O+DiCzmEL+W9sqXxZaqjsj4RJbWFcyPpUZejk=",
|
| 23 |
+
"symbols.json": "rhnF4H8Jsb6I+YPxIVFMgnLcTQJODEXbZ7ge77NZXAU="
|
| 24 |
+
}
|
| 25 |
+
},
|
| 26 |
+
"provenance": {
|
| 27 |
+
"kernel-builder": {
|
| 28 |
+
"version": "0.18.0-dev0",
|
| 29 |
+
"commit": "d054906a13710b811b96697b4635f03020842733",
|
| 30 |
+
"dirty": false
|
| 31 |
+
},
|
| 32 |
+
"kernel": {
|
| 33 |
+
"commit": "fe2bb66fee6c2a77eccf08a0508dc57428d488ed",
|
| 34 |
+
"dirty": false
|
| 35 |
+
}
|
| 36 |
+
}
|
| 37 |
+
}
|
build/torch211-rocm71-x86_64-linux/metadata.json.sigstore
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json", "verificationMaterial":{"certificate":{"rawBytes":"MIIHLTCCBrKgAwIBAgIUEmpsGYHCGCL1f77EXMK1mpdslE0wCgYIKoZIzj0EAwMwNzEVMBMGA1UEChMMc2lnc3RvcmUuZGV2MR4wHAYDVQQDExVzaWdzdG9yZS1pbnRlcm1lZGlhdGUwHhcNMjYxMDE5MjIzMTEzWhcNMjYxMDE5MjI0MTEzWjAAMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE4Fy9kpEGzuCqkOEdVCP+RLOaIvA6RK5RuqsBRqXOQ1oE/UwcaBM8VLmKzmbspKP8LZxu8R+MYRHT7vmO2qJGLKOCBdEwggXNMA4GA1UdDwEB/wQEAwIHgDATBgNVHSUEDDAKBggrBgEFBQcDAzAfBgNVHSMEGDAWgBTf0+nPViQRlvmo2OkoVaLGLhhkPzBrBgNVHREBAf8EYTBfhl1odHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkvLmdpdGh1Yi93b3JrZmxvd3MvYnVpbGQueWFtbEByZWZzL2hlYWRzL21haW4wOQYKKwYBBAGDvzABAQQraHR0cHM6Ly90b2tlbi5hY3Rpb25zLmdpdGh1YnVzZXJjb250ZW50LmNvbTAfBgorBgEEAYO/MAECBBF3b3JrZmxvd19kaXNwYXRjaDA2BgorBgEEAYO/MAEDBChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMBMGCisGAQQBg78wAQQEBUJ1aWxkMCsGCisGAQQBg78wAQUEHWh1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5MB0GCisGAQQBg78wAQYED3JlZnMvaGVhZHMvbWFpbjA7BgorBgEEAYO/MAEIBC0MK2h0dHBzOi8vdG9rZW4uYWN0aW9ucy5naXRodWJ1c2VyY29udGVudC5jb20wbQYKKwYBBAGDvzABCQRfDF1odHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkvLmdpdGh1Yi93b3JrZmxvd3MvYnVpbGQueWFtbEByZWZzL2hlYWRzL21haW4wOAYKKwYBBAGDvzABCgQqDChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMBsGCisGAQQBg78wAQsEDQwLc2VsZi1ob3N0ZWQwQAYKKwYBBAGDvzABDAQyDDBodHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkwOAYKKwYBBAGDvzABDQQqDChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMB8GCisGAQQBg78wAQ4EEQwPcmVmcy9oZWFkcy9tYWluMBoGCisGAQQBg78wAQ8EDAwKMTA3MTQ3NTUyOTAuBgorBgEEAYO/MAEQBCAMHmh0dHBzOi8vZ2l0aHViLmNvbS9odWdnaW5nZmFjZTAYBgorBgEEAYO/MAERBAoMCDI1NzIwNzQzMG0GCisGAQQBg78wARIEXwxdaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5Ly5naXRodWIvd29ya2Zsb3dzL2J1aWxkLnlhbWxAcmVmcy9oZWFkcy9tYWluMDgGCisGAQQBg78wARMEKgwoZmUyYmI2NmZlZTZjMmE3N2VjY2YwOGEwNTA4ZGM1NzQyOGQ0ODhlZDAhBgorBgEEAYO/MAEUBBMMEXdvcmtmbG93X2Rpc3BhdGNoMGQGCisGAQQBg78wARUEVgxUaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5L2FjdGlvbnMvcnVucy8zOTIzOTUxODEwNC9hdHRlbXB0cy8xMBYGCisGAQQBg78wARYECAwGcHVibGljMEYGCisGAQQBg78wARgEOAw2cmVwbzpodWdnaW5nZmFjZS9rZXJuZWxzLWNvbW11bml0eTpyZWY6cmVmcy9oZWFkcy9tYWluMIGKBgorBgEEAdZ5AgQCBHwEegB4AHYA3T0wasbHETJjGR4cmWc3AqJKXrjePK3/h4pygC8p7o4AAAGhVkpQHwAABAMARzBFAiEA5+eh0gx7+UP+mTMZ7dYGtXAGoMr/u3B/1ATobugZirgCIDMCumLOPPvo9Ns0vC8ybrQt7fXH5h98acJfF9aYV1fuMAoGCCqGSM49BAMDA2kAMGYCMQC1gggEuYtV0xjSBQFLnBzb2NdSMXXnrwNQMbOsqeldlpTGoUouHMgT0NboOvp74NcCMQCy79wgE2YbxXDZW7ye5v9a8T72ozdyyAG3+bFixIHj0rx86inTdkuqjr8I90iw9ok="}, "tlogEntries":[{"logIndex":"3327211453", "logId":{"keyId":"wNI9atQGlz+VWfO6LRygH4QUfY/8W4RFwiT5i5WRgB0="}, "kindVersion":{"kind":"hashedrekord", "version":"0.0.1"}, "integratedTime":"1792449073", "inclusionPromise":{"signedEntryTimestamp":"MEQCICqb2FveJcObQ1buwFT+0NvR4ZCa0apWIyoOzyrWsgzYAiB4YVmCelHrY9h3rlc+qWSF+O+hNiQ/syPijjNsHJwahg=="}, "inclusionProof":{"logIndex":"3205307191", "rootHash":"iiOp2twenBLzbfls9348G+Zypx6NFcfs3+o2fG/C/fk=", "treeSize":"3205307194", "hashes":["fquilGZratvzTDiv84GzP8s55UYAN0iUI2kGqK20cXE=", "fugeqQEriw+je6JcoVNkTkv+UD+IzIQtDv/zaj8Quj8=", "4hRzU/5ELB96P1nZkhgwpYYVgFAr1DXDztt20pO7If4=", "rM02OjNrzut1+wWBYgEkyMNTWk3J8j72VYpb5yxWm/g=", "gIcTbwmbW4Jinimt4uWlbE/g8i6p+zi5IXTWev0jxFY=", "iekAzKnn2vj8KKbMbX8Obrd5/3Mxh3/6lsToOqKeuTU=", "AJlvaM7WQKZg0FAjSdh7RzJXyGKaOW5DVz46FbaUPZE=", "t4Yy4kaoiJ3+VosqXw5Ogmg+A7p/E6y8+JhqWbalMbI=", "5egUlZ4Pi0Ix/qLyCDXocsWywJCHqrde/S06vOZvlOM=", "QsXufUpoKtSjKAB19K1G00umQpWvPHlBNuRGFiiVjQQ=", "QIMkwQbwAewhLJd//5FFwI3SnyNQBGjCgJrelfG8vhc=", "Et17CACUv9QN64irUJuBjsrgRGkGnnJ9xacRV/aBdwQ=", "lqgEwkxhDMs/gdIAE4lmjfMwg7nDmv7lda6O0LGliCw=", "HovTPBWzoErvUg1u5sIi3x1ZonZ7yJekVsBrmpomCIA=", "G+nk7mgE5h6dUmv2UY7NDJrY3WvrZY31raaO1AETt+o=", "qlTWNDzERnTBQVcHHXXVPjiUP0SsikGxfcxVlgqVFDo=", "gg9GNApNke3+zNhl4tsWtmIycCFrOMCB3lys6XuSQNQ=", "NlOclldmKPA1tMKIskqHAwuQDYHp31YwpK+OOMrMccc=", "qxzHanAzz57SDdmJe0B7bJK72NTIBbwEMGBBKvDOROw=", "xH/DCseLHr9eKoYT8qsORZK7zVdEGYWHuVtsVrD95wY="], "checkpoint":{"envelope":"rekor.sigstore.dev - 1193050959916656506\n3205307194\niiOp2twenBLzbfls9348G+Zypx6NFcfs3+o2fG/C/fk=\n\n— rekor.sigstore.dev wNI9ajBGAiEA4MrXZ0/pNdLZFmSAyN8SsGiWgHq/FNXsTxStKfSyb5YCIQD4kmm+enQutJHTg7MkOS+ckvQ2VGJuDl8kjANqXlKi6Q==\n"}}, "canonicalizedBody":"eyJhcGlWZXJzaW9uIjoiMC4wLjEiLCJraW5kIjoiaGFzaGVkcmVrb3JkIiwic3BlYyI6eyJkYXRhIjp7Imhhc2giOnsiYWxnb3JpdGhtIjoic2hhMjU2IiwidmFsdWUiOiI1MGE4MDNkMjI4NDhkOGExYThmNmRhYWRmNWJkZTVhYWQ2ZDk3MzUwNjI1OTIwY2E4OTVmYzAyNDM2ZjVkNDQ1In19LCJzaWduYXR1cmUiOnsiY29udGVudCI6Ik1FUUNJQm8rZ3Ftb09qeUcxNVFpUkRRd3d4QTNuMVExVi83dUJrOFRLYlZ5ZEpkOEFpQnFNcC96SlZkditjN1IzSHM2UlFnbE5EQmFhZnJYNzRUZ25TQzJ0dDdTTEE9PSIsInB1YmxpY0tleSI6eyJjb250ZW50IjoiTFMwdExTMUNSVWRKVGlCRFJWSlVTVVpKUTBGVVJTMHRMUzB0Q2sxSlNVaE1WRU5EUW5KTFowRjNTVUpCWjBsVlJXMXdjMGRaU0VOSFEwd3haamMzUlZoTlN6RnRjR1J6YkVVd2QwTm5XVWxMYjFwSmVtb3dSVUYzVFhjS1RucEZWazFDVFVkQk1WVkZRMmhOVFdNeWJHNWpNMUoyWTIxVmRWcEhWakpOVWpSM1NFRlpSRlpSVVVSRmVGWjZZVmRrZW1SSE9YbGFVekZ3WW01U2JBcGpiVEZzV2tkc2FHUkhWWGRJYUdOT1RXcFplRTFFUlRWTmFrbDZUVlJGZWxkb1kwNU5hbGw0VFVSRk5VMXFTVEJOVkVWNlYycEJRVTFHYTNkRmQxbElDa3R2V2tsNmFqQkRRVkZaU1V0dldrbDZhakJFUVZGalJGRm5RVVUwUm5rNWEzQkZSM3AxUTNGclQwVmtWa05RSzFKTVQyRkpka0UyVWtzMVVuVnhjMElLVW5GWVQxRXhiMFV2VlhkallVSk5PRlpNYlV0NmJXSnpjRXRRT0V4YWVIVTRVaXROV1ZKSVZEZDJiVTh5Y1VwSFRFdFBRMEprUlhkbloxaE9UVUUwUndwQk1WVmtSSGRGUWk5M1VVVkJkMGxJWjBSQlZFSm5UbFpJVTFWRlJFUkJTMEpuWjNKQ1owVkdRbEZqUkVGNlFXWkNaMDVXU0ZOTlJVZEVRVmRuUWxSbUNqQXJibEJXYVZGU2JIWnRiekpQYTI5V1lVeEhUR2hvYTFCNlFuSkNaMDVXU0ZKRlFrRm1PRVZaVkVKbWFHd3hiMlJJVW5kamVtOTJUREprY0dSSGFERUtXV2sxYW1JeU1IWmhTRlp1V2pKc2RWb3lXbWhaTWxWMllUSldlV0p0Vm5OamVURnFZakl4ZEdSWE5YQmtTR3QyVEcxa2NHUkhhREZaYVRrellqTktjZ3BhYlhoMlpETk5kbGx1Vm5CaVIxRjFaVmRHZEdKRlFubGFWMXA2VERKb2JGbFhVbnBNTWpGb1lWYzBkMDlSV1V0TGQxbENRa0ZIUkhaNlFVSkJVVkZ5Q21GSVVqQmpTRTAyVEhrNU1HSXlkR3hpYVRWb1dUTlNjR0l5TlhwTWJXUndaRWRvTVZsdVZucGFXRXBxWWpJMU1GcFhOVEJNYlU1MllsUkJaa0puYjNJS1FtZEZSVUZaVHk5TlFVVkRRa0pHTTJJelNuSmFiWGgyWkRFNWEyRllUbmRaV0ZKcVlVUkJNa0puYjNKQ1owVkZRVmxQTDAxQlJVUkNRMmh0V2xSS2FRcFphbGt5V20xV2JFNXRUWGxaVkdNeldsZE9hbHBxUVRSWlZFRXhUVVJvYTFsNlZUTk9SRWswV2tSUk5FOUhWbXROUWsxSFEybHpSMEZSVVVKbk56aDNDa0ZSVVVWQ1ZVb3hZVmQ0YTAxRGMwZERhWE5IUVZGUlFtYzNPSGRCVVZWRlNGZG9NVm95WkhCaWJXUnRXVmRPYkV3eWRHeGpiVFZzWWtoTmRGa3lPWFFLWWxoV2RXRllValZOUWpCSFEybHpSMEZSVVVKbk56aDNRVkZaUlVRelNteGFiazEyWVVkV2FGcElUWFppVjBad1ltcEJOMEpuYjNKQ1owVkZRVmxQTHdwTlFVVkpRa013VFVzeWFEQmtTRUo2VDJrNGRtUkhPWEphVnpSMVdWZE9NR0ZYT1hWamVUVnVZVmhTYjJSWFNqRmpNbFo1V1RJNWRXUkhWblZrUXpWcUNtSXlNSGRpVVZsTFMzZFpRa0pCUjBSMmVrRkNRMUZTWmtSR01XOWtTRkozWTNwdmRrd3laSEJrUjJneFdXazFhbUl5TUhaaFNGWnVXakpzZFZveVdtZ0tXVEpWZG1FeVZubGliVlp6WTNreGFtSXlNWFJrVnpWd1pFaHJka3h0WkhCa1IyZ3hXV2s1TTJJelNuSmFiWGgyWkROTmRsbHVWbkJpUjFGMVpWZEdkQXBpUlVKNVdsZGFla3d5YUd4WlYxSjZUREl4YUdGWE5IZFBRVmxMUzNkWlFrSkJSMFIyZWtGQ1EyZFJjVVJEYUcxYVZFcHBXV3BaTWxwdFZteE9iVTE1Q2xsVVl6TmFWMDVxV21wQk5GbFVRVEZOUkdocldYcFZNMDVFU1RSYVJGRTBUMGRXYTAxQ2MwZERhWE5IUVZGUlFtYzNPSGRCVVhORlJGRjNUR015Vm5NS1dta3hiMkl6VGpCYVYxRjNVVUZaUzB0M1dVSkNRVWRFZG5wQlFrUkJVWGxFUkVKdlpFaFNkMk42YjNaTU1tUndaRWRvTVZscE5XcGlNakIyWVVoV2JncGFNbXgxV2pKYWFGa3lWWFpoTWxaNVltMVdjMk41TVdwaU1qRjBaRmMxY0dSSWEzZFBRVmxMUzNkWlFrSkJSMFIyZWtGQ1JGRlJjVVJEYUcxYVZFcHBDbGxxV1RKYWJWWnNUbTFOZVZsVVl6TmFWMDVxV21wQk5GbFVRVEZOUkdocldYcFZNMDVFU1RSYVJGRTBUMGRXYTAxQ09FZERhWE5IUVZGUlFtYzNPSGNLUVZFMFJVVlJkMUJqYlZadFkzazViMXBYUm10amVUbDBXVmRzZFUxQ2IwZERhWE5IUVZGUlFtYzNPSGRCVVRoRlJFRjNTMDFVUVROTlZGRXpUbFJWZVFwUFZFRjFRbWR2Y2tKblJVVkJXVTh2VFVGRlVVSkRRVTFJYldnd1pFaENlazlwT0haYU1td3dZVWhXYVV4dFRuWmlVemx2WkZka2JtRlhOVzVhYlVacUNscFVRVmxDWjI5eVFtZEZSVUZaVHk5TlFVVlNRa0Z2VFVORVNURk9la2wzVG5wUmVrMUhNRWREYVhOSFFWRlJRbWMzT0hkQlVrbEZXSGQ0WkdGSVVqQUtZMGhOTmt4NU9XNWhXRkp2WkZkSmRWa3lPWFJNTW1neFdqSmtjR0p0WkcxWlYwNXNUREowYkdOdE5XeGlTRTEwV1RJNWRHSllWblZoV0ZJMVRIazFiZ3BoV0ZKdlpGZEpkbVF5T1hsaE1scHpZak5rZWt3eVNqRmhWM2hyVEc1c2FHSlhlRUZqYlZadFkzazViMXBYUm10amVUbDBXVmRzZFUxRVowZERhWE5IQ2tGUlVVSm5OemgzUVZKTlJVdG5kMjlhYlZWNVdXMUpNazV0V214YVZGcHFUVzFGTTA0eVZtcFpNbGwzVDBkRmQwNVVRVFJhUjAweFRucFJlVTlIVVRBS1QwUm9iRnBFUVdoQ1oyOXlRbWRGUlVGWlR5OU5RVVZWUWtKTlRVVllaSFpqYlhSdFlrYzVNMWd5VW5Cak0wSm9aRWRPYjAxSFVVZERhWE5IUVZGUlFncG5OemgzUVZKVlJWWm5lRlZoU0ZJd1kwaE5Oa3g1T1c1aFdGSnZaRmRKZFZreU9YUk1NbWd4V2pKa2NHSnRaRzFaVjA1c1RESjBiR050Tld4aVNFMTBDbGt5T1hSaVdGWjFZVmhTTlV3eVJtcGtSMngyWW01TmRtTnVWblZqZVRoNlQxUkplazlVVlhoUFJFVjNUa001YUdSSVVteGlXRUl3WTNrNGVFMUNXVWNLUTJselIwRlJVVUpuTnpoM1FWSlpSVU5CZDBkalNGWnBZa2RzYWsxRldVZERhWE5IUVZGUlFtYzNPSGRCVW1kRlQwRjNNbU50Vm5kaWVuQnZaRmRrYmdwaFZ6VnVXbTFHYWxwVE9YSmFXRXAxV2xkNGVreFhUblppVnpFeFltMXNNR1ZVY0hsYVYxazJZMjFXYldONU9XOWFWMFpyWTNrNWRGbFhiSFZOU1VkTENrSm5iM0pDWjBWRlFXUmFOVUZuVVVOQ1NIZEZaV2RDTkVGSVdVRXpWREIzWVhOaVNFVlVTbXBIVWpSamJWZGpNMEZ4U2t0WWNtcGxVRXN6TDJnMGNIa0taME00Y0Rkdk5FRkJRVWRvVm10d1VVaDNRVUZDUVUxQlVucENSa0ZwUlVFMUsyVm9NR2Q0Tnl0VlVDdHRWRTFhTjJSWlIzUllRVWR2VFhJdmRUTkNMd294UVZSdlluVm5XbWx5WjBOSlJFMURkVzFNVDFCUWRtODVUbk13ZGtNNGVXSnlVWFEzWmxoSU5XZzVPR0ZqU21aR09XRlpWakZtZFUxQmIwZERRM0ZIQ2xOTk5EbENRVTFFUVRKclFVMUhXVU5OVVVNeFoyZG5SWFZaZEZZd2VHcFRRbEZHVEc1Q2VtSXlUbVJUVFZoWWJuSjNUbEZOWWs5emNXVnNaR3h3VkVjS2IxVnZkVWhOWjFRd1RtSnZUM1p3TnpST1kwTk5VVU41TnpsM1owVXlXV0o0V0VSYVZ6ZDVaVFYyT1dFNFZEY3liM3BrZVhsQlJ6TXJZa1pwZUVsSWFnb3djbmc0Tm1sdVZHUnJkWEZxY2poSk9UQnBkemx2YXowS0xTMHRMUzFGVGtRZ1EwVlNWRWxHU1VOQlZFVXRMUzB0TFFvPSJ9fX19"}], "timestampVerificationData":{"rfc3161Timestamps":[{"signedTimestamp":"MIICyjADAgEAMIICwQYJKoZIhvcNAQcCoIICsjCCAq4CAQMxDTALBglghkgBZQMEAgEwgbgGCyqGSIb3DQEJEAEEoIGoBIGlMIGiAgEBBgkrBgEEAYO/MAIwMTANBglghkgBZQMEAgEFAAQgve8SsvliP+kBGAv5uenHP7txgoqMJQ//kJyYCG+oIZQCFQCFqztCSeTNbThwd1EK8pfaDGvUzxgPMjAyNjEwMTkyMjMxMTNaMAMCAQGgMqQwMC4xFTATBgNVBAoTDHNpZ3N0b3JlLmRldjEVMBMGA1UEAxMMc2lnc3RvcmUtdHNhoAAxggHbMIIB1wIBATBRMDkxFTATBgNVBAoTDHNpZ3N0b3JlLmRldjEgMB4GA1UEAxMXc2lnc3RvcmUtdHNhLXNlbGZzaWduZWQCFDoTVC8MkGHuvMFDL8uKjosqI4sMMAsGCWCGSAFlAwQCAaCB/DAaBgkqhkiG9w0BCQMxDQYLKoZIhvcNAQkQAQQwHAYJKoZIhvcNAQkFMQ8XDTI2MTAxOTIyMzExM1owLwYJKoZIhvcNAQkEMSIEIABz+BxWlSmUQSkAqKq3rF9he+spJMqg2Knzgd2nAeimMIGOBgsqhkiG9w0BCRACLzF/MH0wezB5BCCF+Se8B6tiysO0Q1bBDvyBssaIP9p6uebYcNnROs0FtzBVMD2kOzA5MRUwEwYDVQQKEwxzaWdzdG9yZS5kZXYxIDAeBgNVBAMTF3NpZ3N0b3JlLXRzYS1zZWxmc2lnbmVkAhQ6E1QvDJBh7rzBQy/Lio6LKiOLDDAKBggqhkjOPQQDAgRnMGUCMDCtCa0AeNpbVDBgccxCXaS9qeyqeakv/2BZZVHiqKiCNrXU79GQ6q9FTWcg6JIxMgIxANVyI0gY8+W9UpPT0WZEBk0Fo2APA2MAPz5gH/ksPyYJIeHDeKqZjnLfkEacqtMAYw=="}]}}, "messageSignature":{"messageDigest":{"algorithm":"SHA2_256", "digest":"UKgD0ihI2KGo9tqt9b3lqtbZc1BiWSDKiV/AJDb11EU="}, "signature":"MEQCIBo+gqmoOjyG15QiRDQwwxA3n1Q1V/7uBk8TKbVydJd8AiBqMp/zJVdv+c7R3Hs6RQglNDBaafrX74TgnSC2tt7SLA=="}}
|
build/torch211-rocm71-x86_64-linux/mha.py
ADDED
|
@@ -0,0 +1,332 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""FlashAttention forward entry points backed by aiter's Composable-Kernel FMHA.
|
| 2 |
+
|
| 3 |
+
This is the compiled (HIP / Composable Kernel) counterpart to the Triton
|
| 4 |
+
``aiter-flash-attn`` package. Only the *forward* pass is provided (dense and
|
| 5 |
+
variable-length), including support for learnable attention sinks via
|
| 6 |
+
``window_size[2]`` / ``s_aux`` (e.g. gpt-oss). The op set ships exclusively
|
| 7 |
+
the CK ``mha_fwd`` / ``mha_varlen_fwd`` kernels, so the sink path is always
|
| 8 |
+
taken (no fall-through to a non-sink ASM kernel).
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
from typing import Optional, Tuple, Union
|
| 12 |
+
|
| 13 |
+
import torch
|
| 14 |
+
from torch import Tensor
|
| 15 |
+
|
| 16 |
+
from ._ops import ops
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def _round_multiple(x: int, m: int) -> int:
|
| 20 |
+
return (x + m - 1) // m * m
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def _maybe_pad_headdim(t: Tensor, target: int) -> Tensor:
|
| 24 |
+
og = t.size(-1)
|
| 25 |
+
if og == target:
|
| 26 |
+
return t
|
| 27 |
+
return torch.nn.functional.pad(t, [0, target - og])
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def _unpack_window(window_size) -> Tuple[int, int, int]:
|
| 31 |
+
left = int(window_size[0])
|
| 32 |
+
right = int(window_size[1])
|
| 33 |
+
sink = int(window_size[2]) if len(window_size) == 3 else 0
|
| 34 |
+
return left, right, sink
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def _prep_sink(s_aux: Optional[Tensor], nheads: int, device) -> Optional[Tensor]:
|
| 38 |
+
if s_aux is None:
|
| 39 |
+
return None
|
| 40 |
+
assert s_aux.device == device, "s_aux must be on the same device as q"
|
| 41 |
+
assert s_aux.shape[0] == nheads, "s_aux must have shape [nheads_q]"
|
| 42 |
+
if s_aux.dtype != torch.float32:
|
| 43 |
+
s_aux = s_aux.to(torch.float32)
|
| 44 |
+
return s_aux
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def _effective_sink_size(sink_size: int, s_aux: Optional[Tensor]) -> int:
|
| 48 |
+
# The CK kernel only takes the sink path when sink_size > 0. Callers using
|
| 49 |
+
# the transformers attention interface pass the per-head sink logits as
|
| 50 |
+
# ``s_aux`` but leave ``window_size`` a 2-tuple (sink_size == 0), so enable
|
| 51 |
+
# a single learnable sink automatically when s_aux is given.
|
| 52 |
+
if s_aux is not None and sink_size == 0:
|
| 53 |
+
return 1
|
| 54 |
+
return sink_size
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def flash_attn_func(
|
| 58 |
+
q: Tensor,
|
| 59 |
+
k: Tensor,
|
| 60 |
+
v: Tensor,
|
| 61 |
+
dropout_p: float = 0.0,
|
| 62 |
+
softmax_scale: Optional[float] = None,
|
| 63 |
+
causal: bool = False,
|
| 64 |
+
window_size: Tuple[int, ...] = (-1, -1, 0),
|
| 65 |
+
bias: Optional[Tensor] = None,
|
| 66 |
+
alibi_slopes: Optional[Tensor] = None,
|
| 67 |
+
return_lse: bool = False,
|
| 68 |
+
return_attn_probs: bool = False,
|
| 69 |
+
s_aux: Optional[Tensor] = None,
|
| 70 |
+
**kwargs,
|
| 71 |
+
) -> Union[Tensor, Tuple[Tensor, ...]]:
|
| 72 |
+
"""Dense FlashAttention forward.
|
| 73 |
+
|
| 74 |
+
``q, k, v`` have shape ``(batch, seqlen, nheads, headdim)``. ``window_size``
|
| 75 |
+
is ``(left, right, sink_size)``; ``s_aux`` is an optional ``(nheads_q,)``
|
| 76 |
+
fp32 tensor of learnable sink logits. Extra keyword arguments (e.g. those
|
| 77 |
+
passed by the transformers flash-attention interface) are ignored.
|
| 78 |
+
"""
|
| 79 |
+
if softmax_scale is None:
|
| 80 |
+
softmax_scale = q.shape[-1] ** (-0.5)
|
| 81 |
+
window_size_left, window_size_right, sink_size = _unpack_window(window_size)
|
| 82 |
+
|
| 83 |
+
nheads_q = q.size(2)
|
| 84 |
+
s_aux = _prep_sink(s_aux, nheads_q, q.device)
|
| 85 |
+
sink_size = _effective_sink_size(sink_size, s_aux)
|
| 86 |
+
|
| 87 |
+
head_q_og = q.size(-1)
|
| 88 |
+
head_v_og = v.size(-1)
|
| 89 |
+
head_q = _round_multiple(head_q_og, 8)
|
| 90 |
+
head_v = _round_multiple(head_v_og, 8)
|
| 91 |
+
q = _maybe_pad_headdim(q, head_q)
|
| 92 |
+
k = _maybe_pad_headdim(k, head_q)
|
| 93 |
+
v = _maybe_pad_headdim(v, head_v)
|
| 94 |
+
|
| 95 |
+
return_softmax = return_attn_probs and dropout_p > 0
|
| 96 |
+
out, softmax_lse, S_dmask, _ = ops.mha_fwd(
|
| 97 |
+
q,
|
| 98 |
+
k,
|
| 99 |
+
v,
|
| 100 |
+
dropout_p,
|
| 101 |
+
softmax_scale,
|
| 102 |
+
causal,
|
| 103 |
+
window_size_left,
|
| 104 |
+
window_size_right,
|
| 105 |
+
sink_size,
|
| 106 |
+
return_lse or return_attn_probs,
|
| 107 |
+
return_softmax,
|
| 108 |
+
None, # cu_seqlens_q
|
| 109 |
+
None, # cu_seqlens_kv
|
| 110 |
+
None, # out
|
| 111 |
+
bias,
|
| 112 |
+
alibi_slopes,
|
| 113 |
+
None, # q_descale
|
| 114 |
+
None, # k_descale
|
| 115 |
+
None, # v_descale
|
| 116 |
+
s_aux,
|
| 117 |
+
None, # gen
|
| 118 |
+
)
|
| 119 |
+
|
| 120 |
+
out = out[..., :head_v_og]
|
| 121 |
+
if not (return_lse or return_attn_probs):
|
| 122 |
+
return out
|
| 123 |
+
result = (out,)
|
| 124 |
+
if return_lse:
|
| 125 |
+
result = result + (softmax_lse,)
|
| 126 |
+
if return_attn_probs:
|
| 127 |
+
result = result + (S_dmask,)
|
| 128 |
+
return result
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
def flash_attn_varlen_func(
|
| 132 |
+
q: Tensor,
|
| 133 |
+
k: Tensor,
|
| 134 |
+
v: Tensor,
|
| 135 |
+
cu_seqlens_q: Tensor,
|
| 136 |
+
cu_seqlens_k: Tensor,
|
| 137 |
+
max_seqlen_q: int,
|
| 138 |
+
max_seqlen_k: int,
|
| 139 |
+
dropout_p: float = 0.0,
|
| 140 |
+
softmax_scale: Optional[float] = None,
|
| 141 |
+
causal: bool = False,
|
| 142 |
+
window_size: Tuple[int, ...] = (-1, -1, 0),
|
| 143 |
+
bias: Optional[Tensor] = None,
|
| 144 |
+
alibi_slopes: Optional[Tensor] = None,
|
| 145 |
+
return_lse: bool = False,
|
| 146 |
+
return_attn_probs: bool = False,
|
| 147 |
+
block_table: Optional[Tensor] = None,
|
| 148 |
+
min_seqlen_q: int = 0,
|
| 149 |
+
logits_soft_cap: float = 0.0,
|
| 150 |
+
zero_tensors: bool = False,
|
| 151 |
+
cu_seqlens_q_padded: Optional[Tensor] = None,
|
| 152 |
+
cu_seqlens_k_padded: Optional[Tensor] = None,
|
| 153 |
+
s_aux: Optional[Tensor] = None,
|
| 154 |
+
**kwargs,
|
| 155 |
+
) -> Union[Tensor, Tuple[Tensor, ...]]:
|
| 156 |
+
"""Variable-length / packed FlashAttention forward.
|
| 157 |
+
|
| 158 |
+
``q`` has shape ``(total_q, nheads, headdim)``; ``cu_seqlens_*`` are int32
|
| 159 |
+
cumulative offsets of shape ``(batch + 1,)``. Sinks are supported through
|
| 160 |
+
``window_size[2]`` / ``s_aux`` as in :func:`flash_attn_func`. Extra keyword
|
| 161 |
+
arguments (e.g. from the transformers flash-attention interface) are ignored.
|
| 162 |
+
"""
|
| 163 |
+
if softmax_scale is None:
|
| 164 |
+
softmax_scale = q.shape[-1] ** (-0.5)
|
| 165 |
+
window_size_left, window_size_right, sink_size = _unpack_window(window_size)
|
| 166 |
+
|
| 167 |
+
nheads_q = q.size(1)
|
| 168 |
+
s_aux = _prep_sink(s_aux, nheads_q, q.device)
|
| 169 |
+
sink_size = _effective_sink_size(sink_size, s_aux)
|
| 170 |
+
|
| 171 |
+
head_q_og = q.size(-1)
|
| 172 |
+
head_v_og = v.size(-1)
|
| 173 |
+
head_q = _round_multiple(head_q_og, 8)
|
| 174 |
+
head_v = _round_multiple(head_v_og, 8)
|
| 175 |
+
q = _maybe_pad_headdim(q, head_q)
|
| 176 |
+
k = _maybe_pad_headdim(k, head_q)
|
| 177 |
+
v = _maybe_pad_headdim(v, head_v)
|
| 178 |
+
|
| 179 |
+
return_softmax = return_attn_probs and dropout_p > 0
|
| 180 |
+
out, softmax_lse, S_dmask, _ = ops.mha_varlen_fwd(
|
| 181 |
+
q,
|
| 182 |
+
k,
|
| 183 |
+
v,
|
| 184 |
+
cu_seqlens_q,
|
| 185 |
+
cu_seqlens_k,
|
| 186 |
+
max_seqlen_q,
|
| 187 |
+
max_seqlen_k,
|
| 188 |
+
min_seqlen_q,
|
| 189 |
+
dropout_p,
|
| 190 |
+
softmax_scale,
|
| 191 |
+
logits_soft_cap,
|
| 192 |
+
zero_tensors,
|
| 193 |
+
causal,
|
| 194 |
+
window_size_left,
|
| 195 |
+
window_size_right,
|
| 196 |
+
sink_size,
|
| 197 |
+
return_lse or return_attn_probs,
|
| 198 |
+
return_softmax,
|
| 199 |
+
None, # out
|
| 200 |
+
block_table,
|
| 201 |
+
bias,
|
| 202 |
+
alibi_slopes,
|
| 203 |
+
None, # q_descale
|
| 204 |
+
None, # k_descale
|
| 205 |
+
None, # v_descale
|
| 206 |
+
None, # gen
|
| 207 |
+
cu_seqlens_q_padded,
|
| 208 |
+
cu_seqlens_k_padded,
|
| 209 |
+
s_aux,
|
| 210 |
+
)
|
| 211 |
+
|
| 212 |
+
out = out[..., :head_v_og]
|
| 213 |
+
if not (return_lse or return_attn_probs):
|
| 214 |
+
return out
|
| 215 |
+
result = (out,)
|
| 216 |
+
if return_lse:
|
| 217 |
+
result = result + (softmax_lse,)
|
| 218 |
+
if return_attn_probs:
|
| 219 |
+
result = result + (S_dmask,)
|
| 220 |
+
return result
|
| 221 |
+
|
| 222 |
+
|
| 223 |
+
def _append_to_contiguous_cache(k_cache, v_cache, k, v, cache_seqlens, Sq):
|
| 224 |
+
"""In-place append of new k/v into a contiguous cache at per-batch offsets."""
|
| 225 |
+
B = k_cache.shape[0]
|
| 226 |
+
if isinstance(cache_seqlens, int):
|
| 227 |
+
cache_seqlens = torch.full((B,), cache_seqlens, device=k_cache.device, dtype=torch.long)
|
| 228 |
+
pos = cache_seqlens.view(B, 1).to(torch.long) + torch.arange(Sq, device=k_cache.device).view(1, Sq)
|
| 229 |
+
bidx = torch.arange(B, device=k_cache.device).view(B, 1).expand(B, Sq)
|
| 230 |
+
k_cache[bidx, pos] = k.to(k_cache.dtype)
|
| 231 |
+
v_cache[bidx, pos] = v.to(v_cache.dtype)
|
| 232 |
+
|
| 233 |
+
|
| 234 |
+
def flash_attn_with_kvcache(
|
| 235 |
+
q: Tensor,
|
| 236 |
+
k_cache: Tensor,
|
| 237 |
+
v_cache: Tensor,
|
| 238 |
+
k: Optional[Tensor] = None,
|
| 239 |
+
v: Optional[Tensor] = None,
|
| 240 |
+
rotary_cos: Optional[Tensor] = None,
|
| 241 |
+
rotary_sin: Optional[Tensor] = None,
|
| 242 |
+
cache_seqlens=None,
|
| 243 |
+
block_table: Optional[Tensor] = None,
|
| 244 |
+
softmax_scale: Optional[float] = None,
|
| 245 |
+
causal: bool = True,
|
| 246 |
+
window_size: Tuple[int, ...] = (-1, -1, 0),
|
| 247 |
+
s_aux: Optional[Tensor] = None,
|
| 248 |
+
return_lse: bool = False,
|
| 249 |
+
**kwargs,
|
| 250 |
+
) -> Union[Tensor, Tuple[Tensor, ...]]:
|
| 251 |
+
"""FlashAttention against a KV cache, built on the CK split-KV kernel.
|
| 252 |
+
|
| 253 |
+
``q`` is ``(batch, seqlen_q, nheads, headdim)``. For a contiguous cache,
|
| 254 |
+
``k_cache``/``v_cache`` are ``(batch, seqlen_cache, nheads_k, headdim)`` and
|
| 255 |
+
``cache_seqlens`` (int or ``(batch,)``) is the number of valid cache entries
|
| 256 |
+
*before* appending. If ``k``/``v`` (the new tokens) are given, they are
|
| 257 |
+
written into the cache in place. The query attends to the valid prefix
|
| 258 |
+
(``cache_seqlens`` + ``seqlen_q``) with bottom-right causal alignment.
|
| 259 |
+
|
| 260 |
+
Notes / current limitations:
|
| 261 |
+
- Rotary embedding is not applied here; pass ``q``/``k`` already rotated
|
| 262 |
+
(``rotary_cos``/``rotary_sin`` are rejected).
|
| 263 |
+
- Paged caches are addressed via ``block_table`` (forwarded to the split-KV
|
| 264 |
+
varlen kernel); in-cache append for paged layouts must be done by the
|
| 265 |
+
caller (pass ``k=v=None``).
|
| 266 |
+
"""
|
| 267 |
+
if rotary_cos is not None or rotary_sin is not None:
|
| 268 |
+
raise NotImplementedError(
|
| 269 |
+
"flash_attn_with_kvcache does not apply rotary; rotate q/k beforehand."
|
| 270 |
+
)
|
| 271 |
+
if softmax_scale is None:
|
| 272 |
+
softmax_scale = q.shape[-1] ** (-0.5)
|
| 273 |
+
window_size_left, window_size_right, sink_size = _unpack_window(window_size)
|
| 274 |
+
B, Sq, Hq, D = q.shape
|
| 275 |
+
s_aux = _prep_sink(s_aux, Hq, q.device)
|
| 276 |
+
sink_size = _effective_sink_size(sink_size, s_aux)
|
| 277 |
+
|
| 278 |
+
paged = block_table is not None
|
| 279 |
+
|
| 280 |
+
# ---- append new tokens into the cache (contiguous layout only) ----
|
| 281 |
+
if k is not None and not paged:
|
| 282 |
+
assert cache_seqlens is not None, "cache_seqlens required to append into the cache"
|
| 283 |
+
_append_to_contiguous_cache(k_cache, v_cache, k, v, cache_seqlens, Sq)
|
| 284 |
+
elif k is not None and paged:
|
| 285 |
+
raise NotImplementedError(
|
| 286 |
+
"in-cache append for paged layouts is not supported; append externally "
|
| 287 |
+
"and call with k=v=None."
|
| 288 |
+
)
|
| 289 |
+
|
| 290 |
+
appended = 0 if k is None else Sq
|
| 291 |
+
|
| 292 |
+
# ---- key lengths ----
|
| 293 |
+
if cache_seqlens is None:
|
| 294 |
+
klens = torch.full((B,), k_cache.shape[1], device=q.device, dtype=torch.long)
|
| 295 |
+
elif isinstance(cache_seqlens, int):
|
| 296 |
+
klens = torch.full((B,), cache_seqlens + appended, device=q.device, dtype=torch.long)
|
| 297 |
+
else:
|
| 298 |
+
klens = cache_seqlens.to(q.device, torch.long) + appended
|
| 299 |
+
|
| 300 |
+
uniform = bool((klens == klens[0]).all().item())
|
| 301 |
+
|
| 302 |
+
# ---- fast path: contiguous cache, uniform length -> dense kernel ----
|
| 303 |
+
if not paged and uniform:
|
| 304 |
+
L = int(klens[0].item())
|
| 305 |
+
return flash_attn_func(
|
| 306 |
+
q, k_cache[:, :L], v_cache[:, :L],
|
| 307 |
+
softmax_scale=softmax_scale, causal=causal,
|
| 308 |
+
window_size=(window_size_left, window_size_right, sink_size),
|
| 309 |
+
s_aux=s_aux, return_lse=return_lse,
|
| 310 |
+
)
|
| 311 |
+
|
| 312 |
+
# ---- general path: variable cache lengths -> split-KV varlen kernel ----
|
| 313 |
+
cu_q = torch.arange(0, (B + 1) * Sq, Sq, device=q.device, dtype=torch.int32)
|
| 314 |
+
cu_k = torch.zeros(B + 1, device=q.device, dtype=torch.int32)
|
| 315 |
+
cu_k[1:] = torch.cumsum(klens, 0).to(torch.int32)
|
| 316 |
+
q_packed = q.reshape(B * Sq, Hq, D)
|
| 317 |
+
if paged:
|
| 318 |
+
k_packed, v_packed = k_cache, v_cache # kernel gathers via block_table
|
| 319 |
+
else:
|
| 320 |
+
k_packed = torch.cat([k_cache[b, : int(klens[b])] for b in range(B)], dim=0)
|
| 321 |
+
v_packed = torch.cat([v_cache[b, : int(klens[b])] for b in range(B)], dim=0)
|
| 322 |
+
|
| 323 |
+
out = flash_attn_varlen_func(
|
| 324 |
+
q_packed, k_packed, v_packed, cu_q, cu_k, Sq, int(klens.max().item()),
|
| 325 |
+
softmax_scale=softmax_scale, causal=causal,
|
| 326 |
+
window_size=(window_size_left, window_size_right, sink_size),
|
| 327 |
+
block_table=block_table, s_aux=s_aux, return_lse=return_lse,
|
| 328 |
+
)
|
| 329 |
+
if return_lse:
|
| 330 |
+
out, lse = out
|
| 331 |
+
return out.reshape(B, Sq, Hq, D), lse
|
| 332 |
+
return out.reshape(B, Sq, Hq, D)
|
build/torch211-rocm71-x86_64-linux/symbols.json
ADDED
|
@@ -0,0 +1,349 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": 1,
|
| 3 |
+
"module": "_aiter_flash_attn_ck_rocm_fe2bb66",
|
| 4 |
+
"functions": [
|
| 5 |
+
{
|
| 6 |
+
"name": "flash_attn_func",
|
| 7 |
+
"qualname": "flash_attn_func",
|
| 8 |
+
"module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha",
|
| 9 |
+
"kind": "function",
|
| 10 |
+
"doc": "Dense FlashAttention forward.\n\n``q, k, v`` have shape ``(batch, seqlen, nheads, headdim)``. ``window_size``\nis ``(left, right, sink_size)``; ``s_aux`` is an optional ``(nheads_q,)``\nfp32 tensor of learnable sink logits. Extra keyword arguments (e.g. those\npassed by the transformers flash-attention interface) are ignored.",
|
| 11 |
+
"signature": {
|
| 12 |
+
"parameters": [
|
| 13 |
+
{
|
| 14 |
+
"name": "q",
|
| 15 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 16 |
+
"default": null,
|
| 17 |
+
"annotation": "torch.Tensor"
|
| 18 |
+
},
|
| 19 |
+
{
|
| 20 |
+
"name": "k",
|
| 21 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 22 |
+
"default": null,
|
| 23 |
+
"annotation": "torch.Tensor"
|
| 24 |
+
},
|
| 25 |
+
{
|
| 26 |
+
"name": "v",
|
| 27 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 28 |
+
"default": null,
|
| 29 |
+
"annotation": "torch.Tensor"
|
| 30 |
+
},
|
| 31 |
+
{
|
| 32 |
+
"name": "dropout_p",
|
| 33 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 34 |
+
"default": "0.0",
|
| 35 |
+
"annotation": "float"
|
| 36 |
+
},
|
| 37 |
+
{
|
| 38 |
+
"name": "softmax_scale",
|
| 39 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 40 |
+
"default": "None",
|
| 41 |
+
"annotation": "float | None"
|
| 42 |
+
},
|
| 43 |
+
{
|
| 44 |
+
"name": "causal",
|
| 45 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 46 |
+
"default": "False",
|
| 47 |
+
"annotation": "bool"
|
| 48 |
+
},
|
| 49 |
+
{
|
| 50 |
+
"name": "window_size",
|
| 51 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 52 |
+
"default": "(-1, -1, 0)",
|
| 53 |
+
"annotation": "Tuple[int, ...]"
|
| 54 |
+
},
|
| 55 |
+
{
|
| 56 |
+
"name": "bias",
|
| 57 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 58 |
+
"default": "None",
|
| 59 |
+
"annotation": "torch.Tensor | None"
|
| 60 |
+
},
|
| 61 |
+
{
|
| 62 |
+
"name": "alibi_slopes",
|
| 63 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 64 |
+
"default": "None",
|
| 65 |
+
"annotation": "torch.Tensor | None"
|
| 66 |
+
},
|
| 67 |
+
{
|
| 68 |
+
"name": "return_lse",
|
| 69 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 70 |
+
"default": "False",
|
| 71 |
+
"annotation": "bool"
|
| 72 |
+
},
|
| 73 |
+
{
|
| 74 |
+
"name": "return_attn_probs",
|
| 75 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 76 |
+
"default": "False",
|
| 77 |
+
"annotation": "bool"
|
| 78 |
+
},
|
| 79 |
+
{
|
| 80 |
+
"name": "s_aux",
|
| 81 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 82 |
+
"default": "None",
|
| 83 |
+
"annotation": "torch.Tensor | None"
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"name": "kwargs",
|
| 87 |
+
"kind": "VAR_KEYWORD",
|
| 88 |
+
"default": null,
|
| 89 |
+
"annotation": null
|
| 90 |
+
}
|
| 91 |
+
],
|
| 92 |
+
"return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]"
|
| 93 |
+
}
|
| 94 |
+
},
|
| 95 |
+
{
|
| 96 |
+
"name": "flash_attn_varlen_func",
|
| 97 |
+
"qualname": "flash_attn_varlen_func",
|
| 98 |
+
"module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha",
|
| 99 |
+
"kind": "function",
|
| 100 |
+
"doc": "Variable-length / packed FlashAttention forward.\n\n``q`` has shape ``(total_q, nheads, headdim)``; ``cu_seqlens_*`` are int32\ncumulative offsets of shape ``(batch + 1,)``. Sinks are supported through\n``window_size[2]`` / ``s_aux`` as in :func:`flash_attn_func`. Extra keyword\narguments (e.g. from the transformers flash-attention interface) are ignored.",
|
| 101 |
+
"signature": {
|
| 102 |
+
"parameters": [
|
| 103 |
+
{
|
| 104 |
+
"name": "q",
|
| 105 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 106 |
+
"default": null,
|
| 107 |
+
"annotation": "torch.Tensor"
|
| 108 |
+
},
|
| 109 |
+
{
|
| 110 |
+
"name": "k",
|
| 111 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 112 |
+
"default": null,
|
| 113 |
+
"annotation": "torch.Tensor"
|
| 114 |
+
},
|
| 115 |
+
{
|
| 116 |
+
"name": "v",
|
| 117 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 118 |
+
"default": null,
|
| 119 |
+
"annotation": "torch.Tensor"
|
| 120 |
+
},
|
| 121 |
+
{
|
| 122 |
+
"name": "cu_seqlens_q",
|
| 123 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 124 |
+
"default": null,
|
| 125 |
+
"annotation": "torch.Tensor"
|
| 126 |
+
},
|
| 127 |
+
{
|
| 128 |
+
"name": "cu_seqlens_k",
|
| 129 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 130 |
+
"default": null,
|
| 131 |
+
"annotation": "torch.Tensor"
|
| 132 |
+
},
|
| 133 |
+
{
|
| 134 |
+
"name": "max_seqlen_q",
|
| 135 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 136 |
+
"default": null,
|
| 137 |
+
"annotation": "int"
|
| 138 |
+
},
|
| 139 |
+
{
|
| 140 |
+
"name": "max_seqlen_k",
|
| 141 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 142 |
+
"default": null,
|
| 143 |
+
"annotation": "int"
|
| 144 |
+
},
|
| 145 |
+
{
|
| 146 |
+
"name": "dropout_p",
|
| 147 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 148 |
+
"default": "0.0",
|
| 149 |
+
"annotation": "float"
|
| 150 |
+
},
|
| 151 |
+
{
|
| 152 |
+
"name": "softmax_scale",
|
| 153 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 154 |
+
"default": "None",
|
| 155 |
+
"annotation": "float | None"
|
| 156 |
+
},
|
| 157 |
+
{
|
| 158 |
+
"name": "causal",
|
| 159 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 160 |
+
"default": "False",
|
| 161 |
+
"annotation": "bool"
|
| 162 |
+
},
|
| 163 |
+
{
|
| 164 |
+
"name": "window_size",
|
| 165 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 166 |
+
"default": "(-1, -1, 0)",
|
| 167 |
+
"annotation": "Tuple[int, ...]"
|
| 168 |
+
},
|
| 169 |
+
{
|
| 170 |
+
"name": "bias",
|
| 171 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 172 |
+
"default": "None",
|
| 173 |
+
"annotation": "torch.Tensor | None"
|
| 174 |
+
},
|
| 175 |
+
{
|
| 176 |
+
"name": "alibi_slopes",
|
| 177 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 178 |
+
"default": "None",
|
| 179 |
+
"annotation": "torch.Tensor | None"
|
| 180 |
+
},
|
| 181 |
+
{
|
| 182 |
+
"name": "return_lse",
|
| 183 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 184 |
+
"default": "False",
|
| 185 |
+
"annotation": "bool"
|
| 186 |
+
},
|
| 187 |
+
{
|
| 188 |
+
"name": "return_attn_probs",
|
| 189 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 190 |
+
"default": "False",
|
| 191 |
+
"annotation": "bool"
|
| 192 |
+
},
|
| 193 |
+
{
|
| 194 |
+
"name": "block_table",
|
| 195 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 196 |
+
"default": "None",
|
| 197 |
+
"annotation": "torch.Tensor | None"
|
| 198 |
+
},
|
| 199 |
+
{
|
| 200 |
+
"name": "min_seqlen_q",
|
| 201 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 202 |
+
"default": "0",
|
| 203 |
+
"annotation": "int"
|
| 204 |
+
},
|
| 205 |
+
{
|
| 206 |
+
"name": "logits_soft_cap",
|
| 207 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 208 |
+
"default": "0.0",
|
| 209 |
+
"annotation": "float"
|
| 210 |
+
},
|
| 211 |
+
{
|
| 212 |
+
"name": "zero_tensors",
|
| 213 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 214 |
+
"default": "False",
|
| 215 |
+
"annotation": "bool"
|
| 216 |
+
},
|
| 217 |
+
{
|
| 218 |
+
"name": "cu_seqlens_q_padded",
|
| 219 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 220 |
+
"default": "None",
|
| 221 |
+
"annotation": "torch.Tensor | None"
|
| 222 |
+
},
|
| 223 |
+
{
|
| 224 |
+
"name": "cu_seqlens_k_padded",
|
| 225 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 226 |
+
"default": "None",
|
| 227 |
+
"annotation": "torch.Tensor | None"
|
| 228 |
+
},
|
| 229 |
+
{
|
| 230 |
+
"name": "s_aux",
|
| 231 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 232 |
+
"default": "None",
|
| 233 |
+
"annotation": "torch.Tensor | None"
|
| 234 |
+
},
|
| 235 |
+
{
|
| 236 |
+
"name": "kwargs",
|
| 237 |
+
"kind": "VAR_KEYWORD",
|
| 238 |
+
"default": null,
|
| 239 |
+
"annotation": null
|
| 240 |
+
}
|
| 241 |
+
],
|
| 242 |
+
"return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]"
|
| 243 |
+
}
|
| 244 |
+
},
|
| 245 |
+
{
|
| 246 |
+
"name": "flash_attn_with_kvcache",
|
| 247 |
+
"qualname": "flash_attn_with_kvcache",
|
| 248 |
+
"module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha",
|
| 249 |
+
"kind": "function",
|
| 250 |
+
"doc": "FlashAttention against a KV cache, built on the CK split-KV kernel.\n\n``q`` is ``(batch, seqlen_q, nheads, headdim)``. For a contiguous cache,\n``k_cache``/``v_cache`` are ``(batch, seqlen_cache, nheads_k, headdim)`` and\n``cache_seqlens`` (int or ``(batch,)``) is the number of valid cache entries\n*before* appending. If ``k``/``v`` (the new tokens) are given, they are\nwritten into the cache in place. The query attends to the valid prefix\n(``cache_seqlens`` + ``seqlen_q``) with bottom-right causal alignment.\n\nNotes / current limitations:\n- Rotary embedding is not applied here; pass ``q``/``k`` already rotated\n (``rotary_cos``/``rotary_sin`` are rejected).\n- Paged caches are addressed via ``block_table`` (forwarded to the split-KV\n varlen kernel); in-cache append for paged layouts must be done by the\n caller (pass ``k=v=None``).",
|
| 251 |
+
"signature": {
|
| 252 |
+
"parameters": [
|
| 253 |
+
{
|
| 254 |
+
"name": "q",
|
| 255 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 256 |
+
"default": null,
|
| 257 |
+
"annotation": "torch.Tensor"
|
| 258 |
+
},
|
| 259 |
+
{
|
| 260 |
+
"name": "k_cache",
|
| 261 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 262 |
+
"default": null,
|
| 263 |
+
"annotation": "torch.Tensor"
|
| 264 |
+
},
|
| 265 |
+
{
|
| 266 |
+
"name": "v_cache",
|
| 267 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 268 |
+
"default": null,
|
| 269 |
+
"annotation": "torch.Tensor"
|
| 270 |
+
},
|
| 271 |
+
{
|
| 272 |
+
"name": "k",
|
| 273 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 274 |
+
"default": "None",
|
| 275 |
+
"annotation": "torch.Tensor | None"
|
| 276 |
+
},
|
| 277 |
+
{
|
| 278 |
+
"name": "v",
|
| 279 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 280 |
+
"default": "None",
|
| 281 |
+
"annotation": "torch.Tensor | None"
|
| 282 |
+
},
|
| 283 |
+
{
|
| 284 |
+
"name": "rotary_cos",
|
| 285 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 286 |
+
"default": "None",
|
| 287 |
+
"annotation": "torch.Tensor | None"
|
| 288 |
+
},
|
| 289 |
+
{
|
| 290 |
+
"name": "rotary_sin",
|
| 291 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 292 |
+
"default": "None",
|
| 293 |
+
"annotation": "torch.Tensor | None"
|
| 294 |
+
},
|
| 295 |
+
{
|
| 296 |
+
"name": "cache_seqlens",
|
| 297 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 298 |
+
"default": "None",
|
| 299 |
+
"annotation": null
|
| 300 |
+
},
|
| 301 |
+
{
|
| 302 |
+
"name": "block_table",
|
| 303 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 304 |
+
"default": "None",
|
| 305 |
+
"annotation": "torch.Tensor | None"
|
| 306 |
+
},
|
| 307 |
+
{
|
| 308 |
+
"name": "softmax_scale",
|
| 309 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 310 |
+
"default": "None",
|
| 311 |
+
"annotation": "float | None"
|
| 312 |
+
},
|
| 313 |
+
{
|
| 314 |
+
"name": "causal",
|
| 315 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 316 |
+
"default": "True",
|
| 317 |
+
"annotation": "bool"
|
| 318 |
+
},
|
| 319 |
+
{
|
| 320 |
+
"name": "window_size",
|
| 321 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 322 |
+
"default": "(-1, -1, 0)",
|
| 323 |
+
"annotation": "Tuple[int, ...]"
|
| 324 |
+
},
|
| 325 |
+
{
|
| 326 |
+
"name": "s_aux",
|
| 327 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 328 |
+
"default": "None",
|
| 329 |
+
"annotation": "torch.Tensor | None"
|
| 330 |
+
},
|
| 331 |
+
{
|
| 332 |
+
"name": "return_lse",
|
| 333 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 334 |
+
"default": "False",
|
| 335 |
+
"annotation": "bool"
|
| 336 |
+
},
|
| 337 |
+
{
|
| 338 |
+
"name": "kwargs",
|
| 339 |
+
"kind": "VAR_KEYWORD",
|
| 340 |
+
"default": null,
|
| 341 |
+
"annotation": null
|
| 342 |
+
}
|
| 343 |
+
],
|
| 344 |
+
"return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]"
|
| 345 |
+
}
|
| 346 |
+
}
|
| 347 |
+
],
|
| 348 |
+
"layers": []
|
| 349 |
+
}
|
build/torch211-rocm72-x86_64-linux/__init__.py
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""AITER Composable-Kernel FlashAttention kernels for AMD ROCm (compiled HIP).
|
| 2 |
+
|
| 3 |
+
Compiled (Composable Kernel / HIP) counterpart to the Triton-based
|
| 4 |
+
``aiter-flash-attn`` package, repackaged from the CK FMHA forward path of the
|
| 5 |
+
`ROCm/aiter <https://github.com/ROCm/aiter>`_ project as a self-contained
|
| 6 |
+
Hugging Face Hub kernel. Exposes the FlashAttention forward entry points with
|
| 7 |
+
learnable attention sink support (e.g. gpt-oss).
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
from .mha import (
|
| 11 |
+
flash_attn_func,
|
| 12 |
+
flash_attn_varlen_func,
|
| 13 |
+
flash_attn_with_kvcache,
|
| 14 |
+
)
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
__kernel_metadata__ = {
|
| 18 |
+
"license": "mit",
|
| 19 |
+
}
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
__all__ = [
|
| 23 |
+
"__kernel_metadata__",
|
| 24 |
+
"flash_attn_func",
|
| 25 |
+
"flash_attn_varlen_func",
|
| 26 |
+
"flash_attn_with_kvcache",
|
| 27 |
+
]
|
build/torch211-rocm72-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:3c2f41c2913536638920d0d10d5ab4b4337639df8803362367d4edde7020b610
|
| 3 |
+
size 192185696
|
build/torch211-rocm72-x86_64-linux/_ops.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
from . import _aiter_flash_attn_ck_rocm_fe2bb66
|
| 3 |
+
ops = torch.ops._aiter_flash_attn_ck_rocm_fe2bb66
|
| 4 |
+
|
| 5 |
+
def add_op_namespace_prefix(op_name: str):
|
| 6 |
+
"""
|
| 7 |
+
Prefix op by namespace.
|
| 8 |
+
"""
|
| 9 |
+
return f"_aiter_flash_attn_ck_rocm_fe2bb66::{op_name}"
|
build/torch211-rocm72-x86_64-linux/metadata.json
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "aiter-flash-attn-ck",
|
| 3 |
+
"id": "_aiter_flash_attn_ck_rocm_fe2bb66",
|
| 4 |
+
"version": 1,
|
| 5 |
+
"kernels-minver": "0.14.0",
|
| 6 |
+
"license": "MIT",
|
| 7 |
+
"python-depends": [],
|
| 8 |
+
"kernel-depends": [],
|
| 9 |
+
"backend": {
|
| 10 |
+
"type": "rocm",
|
| 11 |
+
"archs": [
|
| 12 |
+
"gfx942",
|
| 13 |
+
"gfx950"
|
| 14 |
+
]
|
| 15 |
+
},
|
| 16 |
+
"digest": {
|
| 17 |
+
"algorithm": "sha256",
|
| 18 |
+
"files": {
|
| 19 |
+
"__init__.py": "7wj1GTm2bqTLLCyT4xLuAk/+JWYrhINjY/ODMNWhgZ0=",
|
| 20 |
+
"_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so": "PC9BwpE1NmOJINDRDVq0tDN2Od+IAzYjZ9Tt3nAgthA=",
|
| 21 |
+
"_ops.py": "4WkdYDWWJp60zep+oW/5ym9vIeVMNfrgRCkP3o/w7yU=",
|
| 22 |
+
"mha.py": "zCAjY4O+DiCzmEL+W9sqXxZaqjsj4RJbWFcyPpUZejk=",
|
| 23 |
+
"symbols.json": "rhnF4H8Jsb6I+YPxIVFMgnLcTQJODEXbZ7ge77NZXAU="
|
| 24 |
+
}
|
| 25 |
+
},
|
| 26 |
+
"provenance": {
|
| 27 |
+
"kernel-builder": {
|
| 28 |
+
"version": "0.18.0-dev0",
|
| 29 |
+
"commit": "d054906a13710b811b96697b4635f03020842733",
|
| 30 |
+
"dirty": false
|
| 31 |
+
},
|
| 32 |
+
"kernel": {
|
| 33 |
+
"commit": "fe2bb66fee6c2a77eccf08a0508dc57428d488ed",
|
| 34 |
+
"dirty": false
|
| 35 |
+
}
|
| 36 |
+
}
|
| 37 |
+
}
|
build/torch211-rocm72-x86_64-linux/metadata.json.sigstore
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json", "verificationMaterial":{"certificate":{"rawBytes":"MIIHKzCCBrGgAwIBAgIUQTtf3Iyvqs7McLpN4yWfDq9ATiUwCgYIKoZIzj0EAwMwNzEVMBMGA1UEChMMc2lnc3RvcmUuZGV2MR4wHAYDVQQDExVzaWdzdG9yZS1pbnRlcm1lZGlhdGUwHhcNMjYxMDE5MjIzMTE0WhcNMjYxMDE5MjI0MTE0WjAAMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE3t4nw4ZHrBQ3CbTfkKAwLntof/04udWnGPHoBy22SWv0MFnWlFpH3ZxJg5xRTPsjhFBq+fbV9SsvCvDT14AWx6OCBdAwggXMMA4GA1UdDwEB/wQEAwIHgDATBgNVHSUEDDAKBggrBgEFBQcDAzAfBgNVHSMEGDAWgBTf0+nPViQRlvmo2OkoVaLGLhhkPzBrBgNVHREBAf8EYTBfhl1odHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkvLmdpdGh1Yi93b3JrZmxvd3MvYnVpbGQueWFtbEByZWZzL2hlYWRzL21haW4wOQYKKwYBBAGDvzABAQQraHR0cHM6Ly90b2tlbi5hY3Rpb25zLmdpdGh1YnVzZXJjb250ZW50LmNvbTAfBgorBgEEAYO/MAECBBF3b3JrZmxvd19kaXNwYXRjaDA2BgorBgEEAYO/MAEDBChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMBMGCisGAQQBg78wAQQEBUJ1aWxkMCsGCisGAQQBg78wAQUEHWh1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5MB0GCisGAQQBg78wAQYED3JlZnMvaGVhZHMvbWFpbjA7BgorBgEEAYO/MAEIBC0MK2h0dHBzOi8vdG9rZW4uYWN0aW9ucy5naXRodWJ1c2VyY29udGVudC5jb20wbQYKKwYBBAGDvzABCQRfDF1odHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkvLmdpdGh1Yi93b3JrZmxvd3MvYnVpbGQueWFtbEByZWZzL2hlYWRzL21haW4wOAYKKwYBBAGDvzABCgQqDChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMBsGCisGAQQBg78wAQsEDQwLc2VsZi1ob3N0ZWQwQAYKKwYBBAGDvzABDAQyDDBodHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkwOAYKKwYBBAGDvzABDQQqDChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMB8GCisGAQQBg78wAQ4EEQwPcmVmcy9oZWFkcy9tYWluMBoGCisGAQQBg78wAQ8EDAwKMTA3MTQ3NTUyOTAuBgorBgEEAYO/MAEQBCAMHmh0dHBzOi8vZ2l0aHViLmNvbS9odWdnaW5nZmFjZTAYBgorBgEEAYO/MAERBAoMCDI1NzIwNzQzMG0GCisGAQQBg78wARIEXwxdaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5Ly5naXRodWIvd29ya2Zsb3dzL2J1aWxkLnlhbWxAcmVmcy9oZWFkcy9tYWluMDgGCisGAQQBg78wARMEKgwoZmUyYmI2NmZlZTZjMmE3N2VjY2YwOGEwNTA4ZGM1NzQyOGQ0ODhlZDAhBgorBgEEAYO/MAEUBBMMEXdvcmtmbG93X2Rpc3BhdGNoMGQGCisGAQQBg78wARUEVgxUaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5L2FjdGlvbnMvcnVucy8zOTIzOTUxODEwNC9hdHRlbXB0cy8xMBYGCisGAQQBg78wARYECAwGcHVibGljMEYGCisGAQQBg78wARgEOAw2cmVwbzpodWdnaW5nZmFjZS9rZXJuZWxzLWNvbW11bml0eTpyZWY6cmVmcy9oZWFkcy9tYWluMIGJBgorBgEEAdZ5AgQCBHsEeQB3AHUA3T0wasbHETJjGR4cmWc3AqJKXrjePK3/h4pygC8p7o4AAAGhVkpUpwAABAMARjBEAiA1C1YmkJfAITRpzy5HtV9VCMYhGsoyd5hZgpWPpvfVKQIgOVIL0hfBTYj6gt4xl1xqgUEMiH5fkUfg3ZqFxFEBSRAwCgYIKoZIzj0EAwMDaAAwZQIxAJa4QXsPL81L+vXhGGcrZ1zELYnOUe8fmWuz/GLBPTq/v9rdnXKIzxp1A6bIWU8zowIwHGaQvnlFluvNCg6iFIDlDH///B5I/d+ZJwxA3O9O0q7tZggFRCSR0TX4b4K0pfeQ"}, "tlogEntries":[{"logIndex":"3327211583", "logId":{"keyId":"wNI9atQGlz+VWfO6LRygH4QUfY/8W4RFwiT5i5WRgB0="}, "kindVersion":{"kind":"hashedrekord", "version":"0.0.1"}, "integratedTime":"1792449074", "inclusionPromise":{"signedEntryTimestamp":"MEUCIAr7jCNtRJxMlzr1cB+bYl7IKogmxEEEVe2YNbLgJHA8AiEAiDYLQfiwGECJu77/mDsLBPN+ENXHui3Cb6O8P9ck/H0="}, "inclusionProof":{"logIndex":"3205307321", "rootHash":"+fepLxpPOkncVXhGVS4DEMtC08T30oRyfG2VFcMxDGk=", "treeSize":"3205307355", "hashes":["c7VQLPQB2GvvWkc+o6TggbQrbw9Fpr+eCKJewVrBtS8=", "N+iK1dAfNviVQ3nNHztMlyqFa74NPSlTSceuPLmAmTw=", "2wpyyp1ow0otcAGP3cCZwWLmF9FNsuEklXqugfMzXy8=", "JCSa700+gdDu+84caCYaG3zOO7mrMW3EPEh9Uwh91gU=", "mmy8PbTWegE54CQglLSVqGow9D3xL3rqWVCm4og3U2E=", "ZFFNXt17Kn8H1nfr6Yedo+Gde+FkiTYwyKxIfTBJEhs=", "kb9vIWaHg98rThNcZN0YfYNvf4CBr8OBzgh2Qa/FH2A=", "ykqNcyTAH4N/u/MuimGrj8+/P23oqcyJgXQ8Hd4uapU=", "AJlvaM7WQKZg0FAjSdh7RzJXyGKaOW5DVz46FbaUPZE=", "t4Yy4kaoiJ3+VosqXw5Ogmg+A7p/E6y8+JhqWbalMbI=", "5egUlZ4Pi0Ix/qLyCDXocsWywJCHqrde/S06vOZvlOM=", "QsXufUpoKtSjKAB19K1G00umQpWvPHlBNuRGFiiVjQQ=", "QIMkwQbwAewhLJd//5FFwI3SnyNQBGjCgJrelfG8vhc=", "Et17CACUv9QN64irUJuBjsrgRGkGnnJ9xacRV/aBdwQ=", "lqgEwkxhDMs/gdIAE4lmjfMwg7nDmv7lda6O0LGliCw=", "HovTPBWzoErvUg1u5sIi3x1ZonZ7yJekVsBrmpomCIA=", "G+nk7mgE5h6dUmv2UY7NDJrY3WvrZY31raaO1AETt+o=", "qlTWNDzERnTBQVcHHXXVPjiUP0SsikGxfcxVlgqVFDo=", "gg9GNApNke3+zNhl4tsWtmIycCFrOMCB3lys6XuSQNQ=", "NlOclldmKPA1tMKIskqHAwuQDYHp31YwpK+OOMrMccc=", "qxzHanAzz57SDdmJe0B7bJK72NTIBbwEMGBBKvDOROw=", "xH/DCseLHr9eKoYT8qsORZK7zVdEGYWHuVtsVrD95wY="], "checkpoint":{"envelope":"rekor.sigstore.dev - 1193050959916656506\n3205307355\n+fepLxpPOkncVXhGVS4DEMtC08T30oRyfG2VFcMxDGk=\n\n— rekor.sigstore.dev wNI9ajBEAiBdd0/8rQRLHE4cgX8ENMZ0/RCQPKa4qsZYGkjcthEV3AIgf8uLH7bF4jGMwAoMMQPLM1CGWjMYNWqJw3068ntJIiY=\n"}}, "canonicalizedBody":"eyJhcGlWZXJzaW9uIjoiMC4wLjEiLCJraW5kIjoiaGFzaGVkcmVrb3JkIiwic3BlYyI6eyJkYXRhIjp7Imhhc2giOnsiYWxnb3JpdGhtIjoic2hhMjU2IiwidmFsdWUiOiJjOGJkZjdhN2I0MWRlNTdiZDMyY2Q4NzdhNWRmODNkOTJkNTk2Y2MwMWYzYTM2ZTE1NTEyMjIxZjA4NWU2NGYxIn19LCJzaWduYXR1cmUiOnsiY29udGVudCI6Ik1FVUNJUUN6cEsrZGZSeXdLV3RGVVZHVDB5RXFkcDBOdnNBRXU2UEtLWWVNbm0wOXFRSWdJVmNVcUVnUmVJcDdtd0NMOEFsTkFocThZL3hvL012c09mV2RiQ3Z1QVlZPSIsInB1YmxpY0tleSI6eyJjb250ZW50IjoiTFMwdExTMUNSVWRKVGlCRFJWSlVTVVpKUTBGVVJTMHRMUzB0Q2sxSlNVaExla05EUW5KSFowRjNTVUpCWjBsVlVWUjBaak5KZVhaeGN6ZE5ZMHh3VGpSNVYyWkVjVGxCVkdsVmQwTm5XVWxMYjFwSmVtb3dSVUYzVFhjS1RucEZWazFDVFVkQk1WVkZRMmhOVFdNeWJHNWpNMUoyWTIxVmRWcEhWakpOVWpSM1NFRlpSRlpSVVVSRmVGWjZZVmRrZW1SSE9YbGFVekZ3WW01U2JBcGpiVEZzV2tkc2FHUkhWWGRJYUdOT1RXcFplRTFFUlRWTmFrbDZUVlJGTUZkb1kwNU5hbGw0VFVSRk5VMXFTVEJOVkVVd1YycEJRVTFHYTNkRmQxbElDa3R2V2tsNmFqQkRRVkZaU1V0dldrbDZhakJFUVZGalJGRm5RVVV6ZERSdWR6UmFTSEpDVVRORFlsUm1hMHRCZDB4dWRHOW1MekEwZFdSWGJrZFFTRzhLUW5reU1sTlhkakJOUm01WGJFWndTRE5hZUVwbk5YaFNWRkJ6YW1oR1FuRXJabUpXT1ZOemRrTjJSRlF4TkVGWGVEWlBRMEprUVhkbloxaE5UVUUwUndwQk1WVmtSSGRGUWk5M1VVVkJkMGxJWjBSQlZFSm5UbFpJVTFWRlJFUkJTMEpuWjNKQ1owVkdRbEZqUkVGNlFXWkNaMDVXU0ZOTlJVZEVRVmRuUWxSbUNqQXJibEJXYVZGU2JIWnRiekpQYTI5V1lVeEhUR2hvYTFCNlFuSkNaMDVXU0ZKRlFrRm1PRVZaVkVKbWFHd3hiMlJJVW5kamVtOTJUREprY0dSSGFERUtXV2sxYW1JeU1IWmhTRlp1V2pKc2RWb3lXbWhaTWxWMllUSldlV0p0Vm5OamVURnFZakl4ZEdSWE5YQmtTR3QyVEcxa2NHUkhhREZaYVRrellqTktjZ3BhYlhoMlpETk5kbGx1Vm5CaVIxRjFaVmRHZEdKRlFubGFWMXA2VERKb2JGbFhVbnBNTWpGb1lWYzBkMDlSV1V0TGQxbENRa0ZIUkhaNlFVSkJVVkZ5Q21GSVVqQmpTRTAyVEhrNU1HSXlkR3hpYVRWb1dUTlNjR0l5TlhwTWJXUndaRWRvTVZsdVZucGFXRXBxWWpJMU1GcFhOVEJNYlU1MllsUkJaa0puYjNJS1FtZEZSVUZaVHk5TlFVVkRRa0pHTTJJelNuSmFiWGgyWkRFNWEyRllUbmRaV0ZKcVlVUkJNa0puYjNKQ1owVkZRVmxQTDAxQlJVUkNRMmh0V2xSS2FRcFphbGt5V20xV2JFNXRUWGxaVkdNeldsZE9hbHBxUVRSWlZFRXhUVVJvYTFsNlZUTk9SRWswV2tSUk5FOUhWbXROUWsxSFEybHpSMEZSVVVKbk56aDNDa0ZSVVVWQ1ZVb3hZVmQ0YTAxRGMwZERhWE5IUVZGUlFtYzNPSGRCVVZWRlNGZG9NVm95WkhCaWJXUnRXVmRPYkV3eWRHeGpiVFZzWWtoTmRGa3lPWFFLWWxoV2RXRllValZOUWpCSFEybHpSMEZSVVVKbk56aDNRVkZaUlVRelNteGFiazEyWVVkV2FGcElUWFppVjBad1ltcEJOMEpuYjNKQ1owVkZRVmxQTHdwTlFVVkpRa013VFVzeWFEQmtTRUo2VDJrNGRtUkhPWEphVnpSMVdWZE9NR0ZYT1hWamVUVnVZVmhTYjJSWFNqRmpNbFo1V1RJNWRXUkhWblZrUXpWcUNtSXlNSGRpVVZsTFMzZFpRa0pCUjBSMmVrRkNRMUZTWmtSR01XOWtTRkozWTNwdmRrd3laSEJrUjJneFdXazFhbUl5TUhaaFNGWnVXakpzZFZveVdtZ0tXVEpWZG1FeVZubGliVlp6WTNreGFtSXlNWFJrVnpWd1pFaHJka3h0WkhCa1IyZ3hXV2s1TTJJelNuSmFiWGgyWkROTmRsbHVWbkJpUjFGMVpWZEdkQXBpUlVKNVdsZGFla3d5YUd4WlYxSjZUREl4YUdGWE5IZFBRVmxMUzNkWlFrSkJSMFIyZWtGQ1EyZFJjVVJEYUcxYVZFcHBXV3BaTWxwdFZteE9iVTE1Q2xsVVl6TmFWMDVxV21wQk5GbFVRVEZOUkdocldYcFZNMDVFU1RSYVJGRTBUMGRXYTAxQ2MwZERhWE5IUVZGUlFtYzNPSGRCVVhORlJGRjNUR015Vm5NS1dta3hiMkl6VGpCYVYxRjNVVUZaUzB0M1dVSkNRVWRFZG5wQlFrUkJVWGxFUkVKdlpFaFNkMk42YjNaTU1tUndaRWRvTVZscE5XcGlNakIyWVVoV2JncGFNbXgxV2pKYWFGa3lWWFpoTWxaNVltMVdjMk41TVdwaU1qRjBaRmMxY0dSSWEzZFBRVmxMUzNkWlFrSkJSMFIyZWtGQ1JGRlJjVVJEYUcxYVZFcHBDbGxxV1RKYWJWWnNUbTFOZVZsVVl6TmFWMDVxV21wQk5GbFVRVEZOUkdocldYcFZNMDVFU1RSYVJGRTBUMGRXYTAxQ09FZERhWE5IUVZGUlFtYzNPSGNLUVZFMFJVVlJkMUJqYlZadFkzazViMXBYUm10amVUbDBXVmRzZFUxQ2IwZERhWE5IUVZGUlFtYzNPSGRCVVRoRlJFRjNTMDFVUVROTlZGRXpUbFJWZVFwUFZFRjFRbWR2Y2tKblJVVkJXVTh2VFVGRlVVSkRRVTFJYldnd1pFaENlazlwT0haYU1td3dZVWhXYVV4dFRuWmlVemx2WkZka2JtRlhOVzVhYlVacUNscFVRVmxDWjI5eVFtZEZSVUZaVHk5TlFVVlNRa0Z2VFVORVNURk9la2wzVG5wUmVrMUhNRWREYVhOSFFWRlJRbWMzT0hkQlVrbEZXSGQ0WkdGSVVqQUtZMGhOTmt4NU9XNWhXRkp2WkZkSmRWa3lPWFJNTW1neFdqSmtjR0p0WkcxWlYwNXNUREowYkdOdE5XeGlTRTEwV1RJNWRHSllWblZoV0ZJMVRIazFiZ3BoV0ZKdlpGZEpkbVF5T1hsaE1scHpZak5rZWt3eVNqRmhWM2hyVEc1c2FHSlhlRUZqYlZadFkzazViMXBYUm10amVUbDBXVmRzZFUxRVowZERhWE5IQ2tGUlVVSm5OemgzUVZKTlJVdG5kMjlhYlZWNVdXMUpNazV0V214YVZGcHFUVzFGTTA0eVZtcFpNbGwzVDBkRmQwNVVRVFJhUjAweFRucFJlVTlIVVRBS1QwUm9iRnBFUVdoQ1oyOXlRbWRGUlVGWlR5OU5RVVZWUWtKTlRVVllaSFpqYlhSdFlrYzVNMWd5VW5Cak0wSm9aRWRPYjAxSFVVZERhWE5IUVZGUlFncG5OemgzUVZKVlJWWm5lRlZoU0ZJd1kwaE5Oa3g1T1c1aFdGSnZaRmRKZFZreU9YUk1NbWd4V2pKa2NHSnRaRzFaVjA1c1RESjBiR050Tld4aVNFMTBDbGt5T1hSaVdGWjFZVmhTTlV3eVJtcGtSMngyWW01TmRtTnVWblZqZVRoNlQxUkplazlVVlhoUFJFVjNUa001YUdSSVVteGlXRUl3WTNrNGVFMUNXVWNLUTJselIwRlJVVUpuTnpoM1FWSlpSVU5CZDBkalNGWnBZa2RzYWsxRldVZERhWE5IUVZGUlFtYzNPSGRCVW1kRlQwRjNNbU50Vm5kaWVuQnZaRmRrYmdwaFZ6VnVXbTFHYWxwVE9YSmFXRXAxV2xkNGVreFhUblppVnpFeFltMXNNR1ZVY0hsYVYxazJZMjFXYldONU9XOWFWMFpyWTNrNWRGbFhiSFZOU1VkS0NrSm5iM0pDWjBWRlFXUmFOVUZuVVVOQ1NITkZaVkZDTTBGSVZVRXpWREIzWVhOaVNFVlVTbXBIVWpSamJWZGpNMEZ4U2t0WWNtcGxVRXN6TDJnMGNIa0taME00Y0Rkdk5FRkJRVWRvVm10d1ZYQjNRVUZDUVUxQlVtcENSVUZwUVRGRE1WbHRhMHBtUVVsVVVuQjZlVFZJZEZZNVZrTk5XV2hIYzI5NVpEVm9XZ3BuY0ZkUWNIWm1Wa3RSU1dkUFZrbE1NR2htUWxSWmFqWm5kRFI0YkRGNGNXZFZSVTFwU0RWbWExVm1aek5hY1VaNFJrVkNVMUpCZDBObldVbExiMXBKQ25wcU1FVkJkMDFFWVVGQmQxcFJTWGhCU21FMFVWaHpVRXc0TVV3cmRsaG9SMGRqY2xveGVrVk1XVzVQVldVNFptMVhkWG92UjB4Q1VGUnhMM1k1Y21RS2JsaExTWHA0Y0RGQk5tSkpWMVU0ZW05M1NYZElSMkZSZG01c1JteDFkazVEWnpacFJrbEViRVJJTHk4dlFqVkpMMlFyV2twM2VFRXpUemxQTUhFM2RBcGFaMmRHVWtOVFVqQlVXRFJpTkVzd2NHWmxVUW90TFMwdExVVk9SQ0JEUlZKVVNVWkpRMEZVUlMwdExTMHRDZz09In19fX0="}], "timestampVerificationData":{"rfc3161Timestamps":[{"signedTimestamp":"MIICyzADAgEAMIICwgYJKoZIhvcNAQcCoIICszCCAq8CAQMxDTALBglghkgBZQMEAgEwgbgGCyqGSIb3DQEJEAEEoIGoBIGlMIGiAgEBBgkrBgEEAYO/MAIwMTANBglghkgBZQMEAgEFAAQgnxpe49Bca8BoOZiHasRIYzPfojWVj2O6LRUdho/SpeQCFQCVCrq01TcmtZOV7ywJOzs2MCHCpBgPMjAyNjEwMTkyMjMxMTRaMAMCAQGgMqQwMC4xFTATBgNVBAoTDHNpZ3N0b3JlLmRldjEVMBMGA1UEAxMMc2lnc3RvcmUtdHNhoAAxggHcMIIB2AIBATBRMDkxFTATBgNVBAoTDHNpZ3N0b3JlLmRldjEgMB4GA1UEAxMXc2lnc3RvcmUtdHNhLXNlbGZzaWduZWQCFDoTVC8MkGHuvMFDL8uKjosqI4sMMAsGCWCGSAFlAwQCAaCB/DAaBgkqhkiG9w0BCQMxDQYLKoZIhvcNAQkQAQQwHAYJKoZIhvcNAQkFMQ8XDTI2MTAxOTIyMzExNFowLwYJKoZIhvcNAQkEMSIEILuVLXt8AmHMN6KRJ5J/f2uWMscqD7w9RGP4/WIFhtaqMIGOBgsqhkiG9w0BCRACLzF/MH0wezB5BCCF+Se8B6tiysO0Q1bBDvyBssaIP9p6uebYcNnROs0FtzBVMD2kOzA5MRUwEwYDVQQKEwxzaWdzdG9yZS5kZXYxIDAeBgNVBAMTF3NpZ3N0b3JlLXRzYS1zZWxmc2lnbmVkAhQ6E1QvDJBh7rzBQy/Lio6LKiOLDDAKBggqhkjOPQQDAgRoMGYCMQDBQ5gG1/t9qM5NWnE0tWE0k7iAjLVh1LkWH5yuTH5gMTyC1peLgRopwwY6jQrkyr4CMQCEct/P2loSuprwJPAduGECrNug2LYMwl+jfNReEoXLVUVcTHdo4UN53D3t/T0WJfw="}]}}, "messageSignature":{"messageDigest":{"algorithm":"SHA2_256", "digest":"yL33p7Qd5XvTLNh3pd+D2S1ZbMAfOjbhVRIiHwheZPE="}, "signature":"MEUCIQCzpK+dfRywKWtFUVGT0yEqdp0NvsAEu6PKKYeMnm09qQIgIVcUqEgReIp7mwCL8AlNAhq8Y/xo/MvsOfWdbCvuAYY="}}
|
build/torch211-rocm72-x86_64-linux/mha.py
ADDED
|
@@ -0,0 +1,332 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""FlashAttention forward entry points backed by aiter's Composable-Kernel FMHA.
|
| 2 |
+
|
| 3 |
+
This is the compiled (HIP / Composable Kernel) counterpart to the Triton
|
| 4 |
+
``aiter-flash-attn`` package. Only the *forward* pass is provided (dense and
|
| 5 |
+
variable-length), including support for learnable attention sinks via
|
| 6 |
+
``window_size[2]`` / ``s_aux`` (e.g. gpt-oss). The op set ships exclusively
|
| 7 |
+
the CK ``mha_fwd`` / ``mha_varlen_fwd`` kernels, so the sink path is always
|
| 8 |
+
taken (no fall-through to a non-sink ASM kernel).
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
from typing import Optional, Tuple, Union
|
| 12 |
+
|
| 13 |
+
import torch
|
| 14 |
+
from torch import Tensor
|
| 15 |
+
|
| 16 |
+
from ._ops import ops
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def _round_multiple(x: int, m: int) -> int:
|
| 20 |
+
return (x + m - 1) // m * m
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def _maybe_pad_headdim(t: Tensor, target: int) -> Tensor:
|
| 24 |
+
og = t.size(-1)
|
| 25 |
+
if og == target:
|
| 26 |
+
return t
|
| 27 |
+
return torch.nn.functional.pad(t, [0, target - og])
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def _unpack_window(window_size) -> Tuple[int, int, int]:
|
| 31 |
+
left = int(window_size[0])
|
| 32 |
+
right = int(window_size[1])
|
| 33 |
+
sink = int(window_size[2]) if len(window_size) == 3 else 0
|
| 34 |
+
return left, right, sink
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def _prep_sink(s_aux: Optional[Tensor], nheads: int, device) -> Optional[Tensor]:
|
| 38 |
+
if s_aux is None:
|
| 39 |
+
return None
|
| 40 |
+
assert s_aux.device == device, "s_aux must be on the same device as q"
|
| 41 |
+
assert s_aux.shape[0] == nheads, "s_aux must have shape [nheads_q]"
|
| 42 |
+
if s_aux.dtype != torch.float32:
|
| 43 |
+
s_aux = s_aux.to(torch.float32)
|
| 44 |
+
return s_aux
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def _effective_sink_size(sink_size: int, s_aux: Optional[Tensor]) -> int:
|
| 48 |
+
# The CK kernel only takes the sink path when sink_size > 0. Callers using
|
| 49 |
+
# the transformers attention interface pass the per-head sink logits as
|
| 50 |
+
# ``s_aux`` but leave ``window_size`` a 2-tuple (sink_size == 0), so enable
|
| 51 |
+
# a single learnable sink automatically when s_aux is given.
|
| 52 |
+
if s_aux is not None and sink_size == 0:
|
| 53 |
+
return 1
|
| 54 |
+
return sink_size
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def flash_attn_func(
|
| 58 |
+
q: Tensor,
|
| 59 |
+
k: Tensor,
|
| 60 |
+
v: Tensor,
|
| 61 |
+
dropout_p: float = 0.0,
|
| 62 |
+
softmax_scale: Optional[float] = None,
|
| 63 |
+
causal: bool = False,
|
| 64 |
+
window_size: Tuple[int, ...] = (-1, -1, 0),
|
| 65 |
+
bias: Optional[Tensor] = None,
|
| 66 |
+
alibi_slopes: Optional[Tensor] = None,
|
| 67 |
+
return_lse: bool = False,
|
| 68 |
+
return_attn_probs: bool = False,
|
| 69 |
+
s_aux: Optional[Tensor] = None,
|
| 70 |
+
**kwargs,
|
| 71 |
+
) -> Union[Tensor, Tuple[Tensor, ...]]:
|
| 72 |
+
"""Dense FlashAttention forward.
|
| 73 |
+
|
| 74 |
+
``q, k, v`` have shape ``(batch, seqlen, nheads, headdim)``. ``window_size``
|
| 75 |
+
is ``(left, right, sink_size)``; ``s_aux`` is an optional ``(nheads_q,)``
|
| 76 |
+
fp32 tensor of learnable sink logits. Extra keyword arguments (e.g. those
|
| 77 |
+
passed by the transformers flash-attention interface) are ignored.
|
| 78 |
+
"""
|
| 79 |
+
if softmax_scale is None:
|
| 80 |
+
softmax_scale = q.shape[-1] ** (-0.5)
|
| 81 |
+
window_size_left, window_size_right, sink_size = _unpack_window(window_size)
|
| 82 |
+
|
| 83 |
+
nheads_q = q.size(2)
|
| 84 |
+
s_aux = _prep_sink(s_aux, nheads_q, q.device)
|
| 85 |
+
sink_size = _effective_sink_size(sink_size, s_aux)
|
| 86 |
+
|
| 87 |
+
head_q_og = q.size(-1)
|
| 88 |
+
head_v_og = v.size(-1)
|
| 89 |
+
head_q = _round_multiple(head_q_og, 8)
|
| 90 |
+
head_v = _round_multiple(head_v_og, 8)
|
| 91 |
+
q = _maybe_pad_headdim(q, head_q)
|
| 92 |
+
k = _maybe_pad_headdim(k, head_q)
|
| 93 |
+
v = _maybe_pad_headdim(v, head_v)
|
| 94 |
+
|
| 95 |
+
return_softmax = return_attn_probs and dropout_p > 0
|
| 96 |
+
out, softmax_lse, S_dmask, _ = ops.mha_fwd(
|
| 97 |
+
q,
|
| 98 |
+
k,
|
| 99 |
+
v,
|
| 100 |
+
dropout_p,
|
| 101 |
+
softmax_scale,
|
| 102 |
+
causal,
|
| 103 |
+
window_size_left,
|
| 104 |
+
window_size_right,
|
| 105 |
+
sink_size,
|
| 106 |
+
return_lse or return_attn_probs,
|
| 107 |
+
return_softmax,
|
| 108 |
+
None, # cu_seqlens_q
|
| 109 |
+
None, # cu_seqlens_kv
|
| 110 |
+
None, # out
|
| 111 |
+
bias,
|
| 112 |
+
alibi_slopes,
|
| 113 |
+
None, # q_descale
|
| 114 |
+
None, # k_descale
|
| 115 |
+
None, # v_descale
|
| 116 |
+
s_aux,
|
| 117 |
+
None, # gen
|
| 118 |
+
)
|
| 119 |
+
|
| 120 |
+
out = out[..., :head_v_og]
|
| 121 |
+
if not (return_lse or return_attn_probs):
|
| 122 |
+
return out
|
| 123 |
+
result = (out,)
|
| 124 |
+
if return_lse:
|
| 125 |
+
result = result + (softmax_lse,)
|
| 126 |
+
if return_attn_probs:
|
| 127 |
+
result = result + (S_dmask,)
|
| 128 |
+
return result
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
def flash_attn_varlen_func(
|
| 132 |
+
q: Tensor,
|
| 133 |
+
k: Tensor,
|
| 134 |
+
v: Tensor,
|
| 135 |
+
cu_seqlens_q: Tensor,
|
| 136 |
+
cu_seqlens_k: Tensor,
|
| 137 |
+
max_seqlen_q: int,
|
| 138 |
+
max_seqlen_k: int,
|
| 139 |
+
dropout_p: float = 0.0,
|
| 140 |
+
softmax_scale: Optional[float] = None,
|
| 141 |
+
causal: bool = False,
|
| 142 |
+
window_size: Tuple[int, ...] = (-1, -1, 0),
|
| 143 |
+
bias: Optional[Tensor] = None,
|
| 144 |
+
alibi_slopes: Optional[Tensor] = None,
|
| 145 |
+
return_lse: bool = False,
|
| 146 |
+
return_attn_probs: bool = False,
|
| 147 |
+
block_table: Optional[Tensor] = None,
|
| 148 |
+
min_seqlen_q: int = 0,
|
| 149 |
+
logits_soft_cap: float = 0.0,
|
| 150 |
+
zero_tensors: bool = False,
|
| 151 |
+
cu_seqlens_q_padded: Optional[Tensor] = None,
|
| 152 |
+
cu_seqlens_k_padded: Optional[Tensor] = None,
|
| 153 |
+
s_aux: Optional[Tensor] = None,
|
| 154 |
+
**kwargs,
|
| 155 |
+
) -> Union[Tensor, Tuple[Tensor, ...]]:
|
| 156 |
+
"""Variable-length / packed FlashAttention forward.
|
| 157 |
+
|
| 158 |
+
``q`` has shape ``(total_q, nheads, headdim)``; ``cu_seqlens_*`` are int32
|
| 159 |
+
cumulative offsets of shape ``(batch + 1,)``. Sinks are supported through
|
| 160 |
+
``window_size[2]`` / ``s_aux`` as in :func:`flash_attn_func`. Extra keyword
|
| 161 |
+
arguments (e.g. from the transformers flash-attention interface) are ignored.
|
| 162 |
+
"""
|
| 163 |
+
if softmax_scale is None:
|
| 164 |
+
softmax_scale = q.shape[-1] ** (-0.5)
|
| 165 |
+
window_size_left, window_size_right, sink_size = _unpack_window(window_size)
|
| 166 |
+
|
| 167 |
+
nheads_q = q.size(1)
|
| 168 |
+
s_aux = _prep_sink(s_aux, nheads_q, q.device)
|
| 169 |
+
sink_size = _effective_sink_size(sink_size, s_aux)
|
| 170 |
+
|
| 171 |
+
head_q_og = q.size(-1)
|
| 172 |
+
head_v_og = v.size(-1)
|
| 173 |
+
head_q = _round_multiple(head_q_og, 8)
|
| 174 |
+
head_v = _round_multiple(head_v_og, 8)
|
| 175 |
+
q = _maybe_pad_headdim(q, head_q)
|
| 176 |
+
k = _maybe_pad_headdim(k, head_q)
|
| 177 |
+
v = _maybe_pad_headdim(v, head_v)
|
| 178 |
+
|
| 179 |
+
return_softmax = return_attn_probs and dropout_p > 0
|
| 180 |
+
out, softmax_lse, S_dmask, _ = ops.mha_varlen_fwd(
|
| 181 |
+
q,
|
| 182 |
+
k,
|
| 183 |
+
v,
|
| 184 |
+
cu_seqlens_q,
|
| 185 |
+
cu_seqlens_k,
|
| 186 |
+
max_seqlen_q,
|
| 187 |
+
max_seqlen_k,
|
| 188 |
+
min_seqlen_q,
|
| 189 |
+
dropout_p,
|
| 190 |
+
softmax_scale,
|
| 191 |
+
logits_soft_cap,
|
| 192 |
+
zero_tensors,
|
| 193 |
+
causal,
|
| 194 |
+
window_size_left,
|
| 195 |
+
window_size_right,
|
| 196 |
+
sink_size,
|
| 197 |
+
return_lse or return_attn_probs,
|
| 198 |
+
return_softmax,
|
| 199 |
+
None, # out
|
| 200 |
+
block_table,
|
| 201 |
+
bias,
|
| 202 |
+
alibi_slopes,
|
| 203 |
+
None, # q_descale
|
| 204 |
+
None, # k_descale
|
| 205 |
+
None, # v_descale
|
| 206 |
+
None, # gen
|
| 207 |
+
cu_seqlens_q_padded,
|
| 208 |
+
cu_seqlens_k_padded,
|
| 209 |
+
s_aux,
|
| 210 |
+
)
|
| 211 |
+
|
| 212 |
+
out = out[..., :head_v_og]
|
| 213 |
+
if not (return_lse or return_attn_probs):
|
| 214 |
+
return out
|
| 215 |
+
result = (out,)
|
| 216 |
+
if return_lse:
|
| 217 |
+
result = result + (softmax_lse,)
|
| 218 |
+
if return_attn_probs:
|
| 219 |
+
result = result + (S_dmask,)
|
| 220 |
+
return result
|
| 221 |
+
|
| 222 |
+
|
| 223 |
+
def _append_to_contiguous_cache(k_cache, v_cache, k, v, cache_seqlens, Sq):
|
| 224 |
+
"""In-place append of new k/v into a contiguous cache at per-batch offsets."""
|
| 225 |
+
B = k_cache.shape[0]
|
| 226 |
+
if isinstance(cache_seqlens, int):
|
| 227 |
+
cache_seqlens = torch.full((B,), cache_seqlens, device=k_cache.device, dtype=torch.long)
|
| 228 |
+
pos = cache_seqlens.view(B, 1).to(torch.long) + torch.arange(Sq, device=k_cache.device).view(1, Sq)
|
| 229 |
+
bidx = torch.arange(B, device=k_cache.device).view(B, 1).expand(B, Sq)
|
| 230 |
+
k_cache[bidx, pos] = k.to(k_cache.dtype)
|
| 231 |
+
v_cache[bidx, pos] = v.to(v_cache.dtype)
|
| 232 |
+
|
| 233 |
+
|
| 234 |
+
def flash_attn_with_kvcache(
|
| 235 |
+
q: Tensor,
|
| 236 |
+
k_cache: Tensor,
|
| 237 |
+
v_cache: Tensor,
|
| 238 |
+
k: Optional[Tensor] = None,
|
| 239 |
+
v: Optional[Tensor] = None,
|
| 240 |
+
rotary_cos: Optional[Tensor] = None,
|
| 241 |
+
rotary_sin: Optional[Tensor] = None,
|
| 242 |
+
cache_seqlens=None,
|
| 243 |
+
block_table: Optional[Tensor] = None,
|
| 244 |
+
softmax_scale: Optional[float] = None,
|
| 245 |
+
causal: bool = True,
|
| 246 |
+
window_size: Tuple[int, ...] = (-1, -1, 0),
|
| 247 |
+
s_aux: Optional[Tensor] = None,
|
| 248 |
+
return_lse: bool = False,
|
| 249 |
+
**kwargs,
|
| 250 |
+
) -> Union[Tensor, Tuple[Tensor, ...]]:
|
| 251 |
+
"""FlashAttention against a KV cache, built on the CK split-KV kernel.
|
| 252 |
+
|
| 253 |
+
``q`` is ``(batch, seqlen_q, nheads, headdim)``. For a contiguous cache,
|
| 254 |
+
``k_cache``/``v_cache`` are ``(batch, seqlen_cache, nheads_k, headdim)`` and
|
| 255 |
+
``cache_seqlens`` (int or ``(batch,)``) is the number of valid cache entries
|
| 256 |
+
*before* appending. If ``k``/``v`` (the new tokens) are given, they are
|
| 257 |
+
written into the cache in place. The query attends to the valid prefix
|
| 258 |
+
(``cache_seqlens`` + ``seqlen_q``) with bottom-right causal alignment.
|
| 259 |
+
|
| 260 |
+
Notes / current limitations:
|
| 261 |
+
- Rotary embedding is not applied here; pass ``q``/``k`` already rotated
|
| 262 |
+
(``rotary_cos``/``rotary_sin`` are rejected).
|
| 263 |
+
- Paged caches are addressed via ``block_table`` (forwarded to the split-KV
|
| 264 |
+
varlen kernel); in-cache append for paged layouts must be done by the
|
| 265 |
+
caller (pass ``k=v=None``).
|
| 266 |
+
"""
|
| 267 |
+
if rotary_cos is not None or rotary_sin is not None:
|
| 268 |
+
raise NotImplementedError(
|
| 269 |
+
"flash_attn_with_kvcache does not apply rotary; rotate q/k beforehand."
|
| 270 |
+
)
|
| 271 |
+
if softmax_scale is None:
|
| 272 |
+
softmax_scale = q.shape[-1] ** (-0.5)
|
| 273 |
+
window_size_left, window_size_right, sink_size = _unpack_window(window_size)
|
| 274 |
+
B, Sq, Hq, D = q.shape
|
| 275 |
+
s_aux = _prep_sink(s_aux, Hq, q.device)
|
| 276 |
+
sink_size = _effective_sink_size(sink_size, s_aux)
|
| 277 |
+
|
| 278 |
+
paged = block_table is not None
|
| 279 |
+
|
| 280 |
+
# ---- append new tokens into the cache (contiguous layout only) ----
|
| 281 |
+
if k is not None and not paged:
|
| 282 |
+
assert cache_seqlens is not None, "cache_seqlens required to append into the cache"
|
| 283 |
+
_append_to_contiguous_cache(k_cache, v_cache, k, v, cache_seqlens, Sq)
|
| 284 |
+
elif k is not None and paged:
|
| 285 |
+
raise NotImplementedError(
|
| 286 |
+
"in-cache append for paged layouts is not supported; append externally "
|
| 287 |
+
"and call with k=v=None."
|
| 288 |
+
)
|
| 289 |
+
|
| 290 |
+
appended = 0 if k is None else Sq
|
| 291 |
+
|
| 292 |
+
# ---- key lengths ----
|
| 293 |
+
if cache_seqlens is None:
|
| 294 |
+
klens = torch.full((B,), k_cache.shape[1], device=q.device, dtype=torch.long)
|
| 295 |
+
elif isinstance(cache_seqlens, int):
|
| 296 |
+
klens = torch.full((B,), cache_seqlens + appended, device=q.device, dtype=torch.long)
|
| 297 |
+
else:
|
| 298 |
+
klens = cache_seqlens.to(q.device, torch.long) + appended
|
| 299 |
+
|
| 300 |
+
uniform = bool((klens == klens[0]).all().item())
|
| 301 |
+
|
| 302 |
+
# ---- fast path: contiguous cache, uniform length -> dense kernel ----
|
| 303 |
+
if not paged and uniform:
|
| 304 |
+
L = int(klens[0].item())
|
| 305 |
+
return flash_attn_func(
|
| 306 |
+
q, k_cache[:, :L], v_cache[:, :L],
|
| 307 |
+
softmax_scale=softmax_scale, causal=causal,
|
| 308 |
+
window_size=(window_size_left, window_size_right, sink_size),
|
| 309 |
+
s_aux=s_aux, return_lse=return_lse,
|
| 310 |
+
)
|
| 311 |
+
|
| 312 |
+
# ---- general path: variable cache lengths -> split-KV varlen kernel ----
|
| 313 |
+
cu_q = torch.arange(0, (B + 1) * Sq, Sq, device=q.device, dtype=torch.int32)
|
| 314 |
+
cu_k = torch.zeros(B + 1, device=q.device, dtype=torch.int32)
|
| 315 |
+
cu_k[1:] = torch.cumsum(klens, 0).to(torch.int32)
|
| 316 |
+
q_packed = q.reshape(B * Sq, Hq, D)
|
| 317 |
+
if paged:
|
| 318 |
+
k_packed, v_packed = k_cache, v_cache # kernel gathers via block_table
|
| 319 |
+
else:
|
| 320 |
+
k_packed = torch.cat([k_cache[b, : int(klens[b])] for b in range(B)], dim=0)
|
| 321 |
+
v_packed = torch.cat([v_cache[b, : int(klens[b])] for b in range(B)], dim=0)
|
| 322 |
+
|
| 323 |
+
out = flash_attn_varlen_func(
|
| 324 |
+
q_packed, k_packed, v_packed, cu_q, cu_k, Sq, int(klens.max().item()),
|
| 325 |
+
softmax_scale=softmax_scale, causal=causal,
|
| 326 |
+
window_size=(window_size_left, window_size_right, sink_size),
|
| 327 |
+
block_table=block_table, s_aux=s_aux, return_lse=return_lse,
|
| 328 |
+
)
|
| 329 |
+
if return_lse:
|
| 330 |
+
out, lse = out
|
| 331 |
+
return out.reshape(B, Sq, Hq, D), lse
|
| 332 |
+
return out.reshape(B, Sq, Hq, D)
|
build/torch211-rocm72-x86_64-linux/symbols.json
ADDED
|
@@ -0,0 +1,349 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": 1,
|
| 3 |
+
"module": "_aiter_flash_attn_ck_rocm_fe2bb66",
|
| 4 |
+
"functions": [
|
| 5 |
+
{
|
| 6 |
+
"name": "flash_attn_func",
|
| 7 |
+
"qualname": "flash_attn_func",
|
| 8 |
+
"module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha",
|
| 9 |
+
"kind": "function",
|
| 10 |
+
"doc": "Dense FlashAttention forward.\n\n``q, k, v`` have shape ``(batch, seqlen, nheads, headdim)``. ``window_size``\nis ``(left, right, sink_size)``; ``s_aux`` is an optional ``(nheads_q,)``\nfp32 tensor of learnable sink logits. Extra keyword arguments (e.g. those\npassed by the transformers flash-attention interface) are ignored.",
|
| 11 |
+
"signature": {
|
| 12 |
+
"parameters": [
|
| 13 |
+
{
|
| 14 |
+
"name": "q",
|
| 15 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 16 |
+
"default": null,
|
| 17 |
+
"annotation": "torch.Tensor"
|
| 18 |
+
},
|
| 19 |
+
{
|
| 20 |
+
"name": "k",
|
| 21 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 22 |
+
"default": null,
|
| 23 |
+
"annotation": "torch.Tensor"
|
| 24 |
+
},
|
| 25 |
+
{
|
| 26 |
+
"name": "v",
|
| 27 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 28 |
+
"default": null,
|
| 29 |
+
"annotation": "torch.Tensor"
|
| 30 |
+
},
|
| 31 |
+
{
|
| 32 |
+
"name": "dropout_p",
|
| 33 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 34 |
+
"default": "0.0",
|
| 35 |
+
"annotation": "float"
|
| 36 |
+
},
|
| 37 |
+
{
|
| 38 |
+
"name": "softmax_scale",
|
| 39 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 40 |
+
"default": "None",
|
| 41 |
+
"annotation": "float | None"
|
| 42 |
+
},
|
| 43 |
+
{
|
| 44 |
+
"name": "causal",
|
| 45 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 46 |
+
"default": "False",
|
| 47 |
+
"annotation": "bool"
|
| 48 |
+
},
|
| 49 |
+
{
|
| 50 |
+
"name": "window_size",
|
| 51 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 52 |
+
"default": "(-1, -1, 0)",
|
| 53 |
+
"annotation": "Tuple[int, ...]"
|
| 54 |
+
},
|
| 55 |
+
{
|
| 56 |
+
"name": "bias",
|
| 57 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 58 |
+
"default": "None",
|
| 59 |
+
"annotation": "torch.Tensor | None"
|
| 60 |
+
},
|
| 61 |
+
{
|
| 62 |
+
"name": "alibi_slopes",
|
| 63 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 64 |
+
"default": "None",
|
| 65 |
+
"annotation": "torch.Tensor | None"
|
| 66 |
+
},
|
| 67 |
+
{
|
| 68 |
+
"name": "return_lse",
|
| 69 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 70 |
+
"default": "False",
|
| 71 |
+
"annotation": "bool"
|
| 72 |
+
},
|
| 73 |
+
{
|
| 74 |
+
"name": "return_attn_probs",
|
| 75 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 76 |
+
"default": "False",
|
| 77 |
+
"annotation": "bool"
|
| 78 |
+
},
|
| 79 |
+
{
|
| 80 |
+
"name": "s_aux",
|
| 81 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 82 |
+
"default": "None",
|
| 83 |
+
"annotation": "torch.Tensor | None"
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"name": "kwargs",
|
| 87 |
+
"kind": "VAR_KEYWORD",
|
| 88 |
+
"default": null,
|
| 89 |
+
"annotation": null
|
| 90 |
+
}
|
| 91 |
+
],
|
| 92 |
+
"return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]"
|
| 93 |
+
}
|
| 94 |
+
},
|
| 95 |
+
{
|
| 96 |
+
"name": "flash_attn_varlen_func",
|
| 97 |
+
"qualname": "flash_attn_varlen_func",
|
| 98 |
+
"module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha",
|
| 99 |
+
"kind": "function",
|
| 100 |
+
"doc": "Variable-length / packed FlashAttention forward.\n\n``q`` has shape ``(total_q, nheads, headdim)``; ``cu_seqlens_*`` are int32\ncumulative offsets of shape ``(batch + 1,)``. Sinks are supported through\n``window_size[2]`` / ``s_aux`` as in :func:`flash_attn_func`. Extra keyword\narguments (e.g. from the transformers flash-attention interface) are ignored.",
|
| 101 |
+
"signature": {
|
| 102 |
+
"parameters": [
|
| 103 |
+
{
|
| 104 |
+
"name": "q",
|
| 105 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 106 |
+
"default": null,
|
| 107 |
+
"annotation": "torch.Tensor"
|
| 108 |
+
},
|
| 109 |
+
{
|
| 110 |
+
"name": "k",
|
| 111 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 112 |
+
"default": null,
|
| 113 |
+
"annotation": "torch.Tensor"
|
| 114 |
+
},
|
| 115 |
+
{
|
| 116 |
+
"name": "v",
|
| 117 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 118 |
+
"default": null,
|
| 119 |
+
"annotation": "torch.Tensor"
|
| 120 |
+
},
|
| 121 |
+
{
|
| 122 |
+
"name": "cu_seqlens_q",
|
| 123 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 124 |
+
"default": null,
|
| 125 |
+
"annotation": "torch.Tensor"
|
| 126 |
+
},
|
| 127 |
+
{
|
| 128 |
+
"name": "cu_seqlens_k",
|
| 129 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 130 |
+
"default": null,
|
| 131 |
+
"annotation": "torch.Tensor"
|
| 132 |
+
},
|
| 133 |
+
{
|
| 134 |
+
"name": "max_seqlen_q",
|
| 135 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 136 |
+
"default": null,
|
| 137 |
+
"annotation": "int"
|
| 138 |
+
},
|
| 139 |
+
{
|
| 140 |
+
"name": "max_seqlen_k",
|
| 141 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 142 |
+
"default": null,
|
| 143 |
+
"annotation": "int"
|
| 144 |
+
},
|
| 145 |
+
{
|
| 146 |
+
"name": "dropout_p",
|
| 147 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 148 |
+
"default": "0.0",
|
| 149 |
+
"annotation": "float"
|
| 150 |
+
},
|
| 151 |
+
{
|
| 152 |
+
"name": "softmax_scale",
|
| 153 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 154 |
+
"default": "None",
|
| 155 |
+
"annotation": "float | None"
|
| 156 |
+
},
|
| 157 |
+
{
|
| 158 |
+
"name": "causal",
|
| 159 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 160 |
+
"default": "False",
|
| 161 |
+
"annotation": "bool"
|
| 162 |
+
},
|
| 163 |
+
{
|
| 164 |
+
"name": "window_size",
|
| 165 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 166 |
+
"default": "(-1, -1, 0)",
|
| 167 |
+
"annotation": "Tuple[int, ...]"
|
| 168 |
+
},
|
| 169 |
+
{
|
| 170 |
+
"name": "bias",
|
| 171 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 172 |
+
"default": "None",
|
| 173 |
+
"annotation": "torch.Tensor | None"
|
| 174 |
+
},
|
| 175 |
+
{
|
| 176 |
+
"name": "alibi_slopes",
|
| 177 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 178 |
+
"default": "None",
|
| 179 |
+
"annotation": "torch.Tensor | None"
|
| 180 |
+
},
|
| 181 |
+
{
|
| 182 |
+
"name": "return_lse",
|
| 183 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 184 |
+
"default": "False",
|
| 185 |
+
"annotation": "bool"
|
| 186 |
+
},
|
| 187 |
+
{
|
| 188 |
+
"name": "return_attn_probs",
|
| 189 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 190 |
+
"default": "False",
|
| 191 |
+
"annotation": "bool"
|
| 192 |
+
},
|
| 193 |
+
{
|
| 194 |
+
"name": "block_table",
|
| 195 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 196 |
+
"default": "None",
|
| 197 |
+
"annotation": "torch.Tensor | None"
|
| 198 |
+
},
|
| 199 |
+
{
|
| 200 |
+
"name": "min_seqlen_q",
|
| 201 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 202 |
+
"default": "0",
|
| 203 |
+
"annotation": "int"
|
| 204 |
+
},
|
| 205 |
+
{
|
| 206 |
+
"name": "logits_soft_cap",
|
| 207 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 208 |
+
"default": "0.0",
|
| 209 |
+
"annotation": "float"
|
| 210 |
+
},
|
| 211 |
+
{
|
| 212 |
+
"name": "zero_tensors",
|
| 213 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 214 |
+
"default": "False",
|
| 215 |
+
"annotation": "bool"
|
| 216 |
+
},
|
| 217 |
+
{
|
| 218 |
+
"name": "cu_seqlens_q_padded",
|
| 219 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 220 |
+
"default": "None",
|
| 221 |
+
"annotation": "torch.Tensor | None"
|
| 222 |
+
},
|
| 223 |
+
{
|
| 224 |
+
"name": "cu_seqlens_k_padded",
|
| 225 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 226 |
+
"default": "None",
|
| 227 |
+
"annotation": "torch.Tensor | None"
|
| 228 |
+
},
|
| 229 |
+
{
|
| 230 |
+
"name": "s_aux",
|
| 231 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 232 |
+
"default": "None",
|
| 233 |
+
"annotation": "torch.Tensor | None"
|
| 234 |
+
},
|
| 235 |
+
{
|
| 236 |
+
"name": "kwargs",
|
| 237 |
+
"kind": "VAR_KEYWORD",
|
| 238 |
+
"default": null,
|
| 239 |
+
"annotation": null
|
| 240 |
+
}
|
| 241 |
+
],
|
| 242 |
+
"return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]"
|
| 243 |
+
}
|
| 244 |
+
},
|
| 245 |
+
{
|
| 246 |
+
"name": "flash_attn_with_kvcache",
|
| 247 |
+
"qualname": "flash_attn_with_kvcache",
|
| 248 |
+
"module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha",
|
| 249 |
+
"kind": "function",
|
| 250 |
+
"doc": "FlashAttention against a KV cache, built on the CK split-KV kernel.\n\n``q`` is ``(batch, seqlen_q, nheads, headdim)``. For a contiguous cache,\n``k_cache``/``v_cache`` are ``(batch, seqlen_cache, nheads_k, headdim)`` and\n``cache_seqlens`` (int or ``(batch,)``) is the number of valid cache entries\n*before* appending. If ``k``/``v`` (the new tokens) are given, they are\nwritten into the cache in place. The query attends to the valid prefix\n(``cache_seqlens`` + ``seqlen_q``) with bottom-right causal alignment.\n\nNotes / current limitations:\n- Rotary embedding is not applied here; pass ``q``/``k`` already rotated\n (``rotary_cos``/``rotary_sin`` are rejected).\n- Paged caches are addressed via ``block_table`` (forwarded to the split-KV\n varlen kernel); in-cache append for paged layouts must be done by the\n caller (pass ``k=v=None``).",
|
| 251 |
+
"signature": {
|
| 252 |
+
"parameters": [
|
| 253 |
+
{
|
| 254 |
+
"name": "q",
|
| 255 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 256 |
+
"default": null,
|
| 257 |
+
"annotation": "torch.Tensor"
|
| 258 |
+
},
|
| 259 |
+
{
|
| 260 |
+
"name": "k_cache",
|
| 261 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 262 |
+
"default": null,
|
| 263 |
+
"annotation": "torch.Tensor"
|
| 264 |
+
},
|
| 265 |
+
{
|
| 266 |
+
"name": "v_cache",
|
| 267 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 268 |
+
"default": null,
|
| 269 |
+
"annotation": "torch.Tensor"
|
| 270 |
+
},
|
| 271 |
+
{
|
| 272 |
+
"name": "k",
|
| 273 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 274 |
+
"default": "None",
|
| 275 |
+
"annotation": "torch.Tensor | None"
|
| 276 |
+
},
|
| 277 |
+
{
|
| 278 |
+
"name": "v",
|
| 279 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 280 |
+
"default": "None",
|
| 281 |
+
"annotation": "torch.Tensor | None"
|
| 282 |
+
},
|
| 283 |
+
{
|
| 284 |
+
"name": "rotary_cos",
|
| 285 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 286 |
+
"default": "None",
|
| 287 |
+
"annotation": "torch.Tensor | None"
|
| 288 |
+
},
|
| 289 |
+
{
|
| 290 |
+
"name": "rotary_sin",
|
| 291 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 292 |
+
"default": "None",
|
| 293 |
+
"annotation": "torch.Tensor | None"
|
| 294 |
+
},
|
| 295 |
+
{
|
| 296 |
+
"name": "cache_seqlens",
|
| 297 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 298 |
+
"default": "None",
|
| 299 |
+
"annotation": null
|
| 300 |
+
},
|
| 301 |
+
{
|
| 302 |
+
"name": "block_table",
|
| 303 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 304 |
+
"default": "None",
|
| 305 |
+
"annotation": "torch.Tensor | None"
|
| 306 |
+
},
|
| 307 |
+
{
|
| 308 |
+
"name": "softmax_scale",
|
| 309 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 310 |
+
"default": "None",
|
| 311 |
+
"annotation": "float | None"
|
| 312 |
+
},
|
| 313 |
+
{
|
| 314 |
+
"name": "causal",
|
| 315 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 316 |
+
"default": "True",
|
| 317 |
+
"annotation": "bool"
|
| 318 |
+
},
|
| 319 |
+
{
|
| 320 |
+
"name": "window_size",
|
| 321 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 322 |
+
"default": "(-1, -1, 0)",
|
| 323 |
+
"annotation": "Tuple[int, ...]"
|
| 324 |
+
},
|
| 325 |
+
{
|
| 326 |
+
"name": "s_aux",
|
| 327 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 328 |
+
"default": "None",
|
| 329 |
+
"annotation": "torch.Tensor | None"
|
| 330 |
+
},
|
| 331 |
+
{
|
| 332 |
+
"name": "return_lse",
|
| 333 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 334 |
+
"default": "False",
|
| 335 |
+
"annotation": "bool"
|
| 336 |
+
},
|
| 337 |
+
{
|
| 338 |
+
"name": "kwargs",
|
| 339 |
+
"kind": "VAR_KEYWORD",
|
| 340 |
+
"default": null,
|
| 341 |
+
"annotation": null
|
| 342 |
+
}
|
| 343 |
+
],
|
| 344 |
+
"return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]"
|
| 345 |
+
}
|
| 346 |
+
}
|
| 347 |
+
],
|
| 348 |
+
"layers": []
|
| 349 |
+
}
|
build/torch212-rocm71-x86_64-linux/__init__.py
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""AITER Composable-Kernel FlashAttention kernels for AMD ROCm (compiled HIP).
|
| 2 |
+
|
| 3 |
+
Compiled (Composable Kernel / HIP) counterpart to the Triton-based
|
| 4 |
+
``aiter-flash-attn`` package, repackaged from the CK FMHA forward path of the
|
| 5 |
+
`ROCm/aiter <https://github.com/ROCm/aiter>`_ project as a self-contained
|
| 6 |
+
Hugging Face Hub kernel. Exposes the FlashAttention forward entry points with
|
| 7 |
+
learnable attention sink support (e.g. gpt-oss).
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
from .mha import (
|
| 11 |
+
flash_attn_func,
|
| 12 |
+
flash_attn_varlen_func,
|
| 13 |
+
flash_attn_with_kvcache,
|
| 14 |
+
)
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
__kernel_metadata__ = {
|
| 18 |
+
"license": "mit",
|
| 19 |
+
}
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
__all__ = [
|
| 23 |
+
"__kernel_metadata__",
|
| 24 |
+
"flash_attn_func",
|
| 25 |
+
"flash_attn_varlen_func",
|
| 26 |
+
"flash_attn_with_kvcache",
|
| 27 |
+
]
|
build/torch212-rocm71-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:78095dfb6de1a2d229d40cdf34e125605bb33d7fa467134d236cb51df66eaf9b
|
| 3 |
+
size 201962680
|
build/torch212-rocm71-x86_64-linux/_ops.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
from . import _aiter_flash_attn_ck_rocm_fe2bb66
|
| 3 |
+
ops = torch.ops._aiter_flash_attn_ck_rocm_fe2bb66
|
| 4 |
+
|
| 5 |
+
def add_op_namespace_prefix(op_name: str):
|
| 6 |
+
"""
|
| 7 |
+
Prefix op by namespace.
|
| 8 |
+
"""
|
| 9 |
+
return f"_aiter_flash_attn_ck_rocm_fe2bb66::{op_name}"
|
build/torch212-rocm71-x86_64-linux/metadata.json
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "aiter-flash-attn-ck",
|
| 3 |
+
"id": "_aiter_flash_attn_ck_rocm_fe2bb66",
|
| 4 |
+
"version": 1,
|
| 5 |
+
"kernels-minver": "0.14.0",
|
| 6 |
+
"license": "MIT",
|
| 7 |
+
"python-depends": [],
|
| 8 |
+
"kernel-depends": [],
|
| 9 |
+
"backend": {
|
| 10 |
+
"type": "rocm",
|
| 11 |
+
"archs": [
|
| 12 |
+
"gfx942",
|
| 13 |
+
"gfx950"
|
| 14 |
+
]
|
| 15 |
+
},
|
| 16 |
+
"digest": {
|
| 17 |
+
"algorithm": "sha256",
|
| 18 |
+
"files": {
|
| 19 |
+
"__init__.py": "7wj1GTm2bqTLLCyT4xLuAk/+JWYrhINjY/ODMNWhgZ0=",
|
| 20 |
+
"_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so": "eAld+23hotIp1AzfNOElYFuzPX+kZxNNI2y1HfZur5s=",
|
| 21 |
+
"_ops.py": "4WkdYDWWJp60zep+oW/5ym9vIeVMNfrgRCkP3o/w7yU=",
|
| 22 |
+
"mha.py": "zCAjY4O+DiCzmEL+W9sqXxZaqjsj4RJbWFcyPpUZejk=",
|
| 23 |
+
"symbols.json": "rhnF4H8Jsb6I+YPxIVFMgnLcTQJODEXbZ7ge77NZXAU="
|
| 24 |
+
}
|
| 25 |
+
},
|
| 26 |
+
"provenance": {
|
| 27 |
+
"kernel-builder": {
|
| 28 |
+
"version": "0.18.0-dev0",
|
| 29 |
+
"commit": "d054906a13710b811b96697b4635f03020842733",
|
| 30 |
+
"dirty": false
|
| 31 |
+
},
|
| 32 |
+
"kernel": {
|
| 33 |
+
"commit": "fe2bb66fee6c2a77eccf08a0508dc57428d488ed",
|
| 34 |
+
"dirty": false
|
| 35 |
+
}
|
| 36 |
+
}
|
| 37 |
+
}
|
build/torch212-rocm71-x86_64-linux/metadata.json.sigstore
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json", "verificationMaterial":{"certificate":{"rawBytes":"MIIHKzCCBrKgAwIBAgIUNCpKwqkj9HF7/3kzrniINKWzzUMwCgYIKoZIzj0EAwMwNzEVMBMGA1UEChMMc2lnc3RvcmUuZGV2MR4wHAYDVQQDExVzaWdzdG9yZS1pbnRlcm1lZGlhdGUwHhcNMjYxMDE5MjIzMTE1WhcNMjYxMDE5MjI0MTE1WjAAMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE3edsuwnJ7jqqemLx4AZlgRX5HxhcWRE6INRFuuHbmZJClfx/ipjms6nRAaXEXcRASW3lKsIpYeSr/T1KEb00/6OCBdEwggXNMA4GA1UdDwEB/wQEAwIHgDATBgNVHSUEDDAKBggrBgEFBQcDAzAfBgNVHSMEGDAWgBTf0+nPViQRlvmo2OkoVaLGLhhkPzBrBgNVHREBAf8EYTBfhl1odHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkvLmdpdGh1Yi93b3JrZmxvd3MvYnVpbGQueWFtbEByZWZzL2hlYWRzL21haW4wOQYKKwYBBAGDvzABAQQraHR0cHM6Ly90b2tlbi5hY3Rpb25zLmdpdGh1YnVzZXJjb250ZW50LmNvbTAfBgorBgEEAYO/MAECBBF3b3JrZmxvd19kaXNwYXRjaDA2BgorBgEEAYO/MAEDBChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMBMGCisGAQQBg78wAQQEBUJ1aWxkMCsGCisGAQQBg78wAQUEHWh1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5MB0GCisGAQQBg78wAQYED3JlZnMvaGVhZHMvbWFpbjA7BgorBgEEAYO/MAEIBC0MK2h0dHBzOi8vdG9rZW4uYWN0aW9ucy5naXRodWJ1c2VyY29udGVudC5jb20wbQYKKwYBBAGDvzABCQRfDF1odHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkvLmdpdGh1Yi93b3JrZmxvd3MvYnVpbGQueWFtbEByZWZzL2hlYWRzL21haW4wOAYKKwYBBAGDvzABCgQqDChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMBsGCisGAQQBg78wAQsEDQwLc2VsZi1ob3N0ZWQwQAYKKwYBBAGDvzABDAQyDDBodHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkwOAYKKwYBBAGDvzABDQQqDChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMB8GCisGAQQBg78wAQ4EEQwPcmVmcy9oZWFkcy9tYWluMBoGCisGAQQBg78wAQ8EDAwKMTA3MTQ3NTUyOTAuBgorBgEEAYO/MAEQBCAMHmh0dHBzOi8vZ2l0aHViLmNvbS9odWdnaW5nZmFjZTAYBgorBgEEAYO/MAERBAoMCDI1NzIwNzQzMG0GCisGAQQBg78wARIEXwxdaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5Ly5naXRodWIvd29ya2Zsb3dzL2J1aWxkLnlhbWxAcmVmcy9oZWFkcy9tYWluMDgGCisGAQQBg78wARMEKgwoZmUyYmI2NmZlZTZjMmE3N2VjY2YwOGEwNTA4ZGM1NzQyOGQ0ODhlZDAhBgorBgEEAYO/MAEUBBMMEXdvcmtmbG93X2Rpc3BhdGNoMGQGCisGAQQBg78wARUEVgxUaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5L2FjdGlvbnMvcnVucy8zOTIzOTUxODEwNC9hdHRlbXB0cy8xMBYGCisGAQQBg78wARYECAwGcHVibGljMEYGCisGAQQBg78wARgEOAw2cmVwbzpodWdnaW5nZmFjZS9rZXJuZWxzLWNvbW11bml0eTpyZWY6cmVmcy9oZWFkcy9tYWluMIGKBgorBgEEAdZ5AgQCBHwEegB4AHYA3T0wasbHETJjGR4cmWc3AqJKXrjePK3/h4pygC8p7o4AAAGhVkpa6AAABAMARzBFAiEApQwCCX96bimcS6xRJKWJBsUiSoDvd+Pk/QYYFcktSVkCIEJ9/2rGZ8klXslytVkeKFQzr+e4l7NootT2Du8uVhacMAoGCCqGSM49BAMDA2cAMGQCMBAEOmqdi0D8R2V1qMg4/OyrvTJWu4oal0KEefWlu6GF7soePSHkA07ZyEiste4BDgIwf5dWNp7viPgDNSfubu6HszrI/4FgaMp5hPAHf5f//UA5nMbgENnYVvxWFKBCVZ8j"}, "tlogEntries":[{"logIndex":"3327211868", "logId":{"keyId":"wNI9atQGlz+VWfO6LRygH4QUfY/8W4RFwiT5i5WRgB0="}, "kindVersion":{"kind":"hashedrekord", "version":"0.0.1"}, "integratedTime":"1792449076", "inclusionPromise":{"signedEntryTimestamp":"MEUCIDMDUHqwsYRzaWixuHGJjWHFJ4/iRt+2UmGi39CfIy1dAiEA+wYw2O5mzlJ9y2j5cF2T4wEuFqjVZysBRi7ItBwq/98="}, "inclusionProof":{"logIndex":"3205307606", "rootHash":"0n+xNJnLMgp7sozEH6SRp+aDmxlicUgusBhZlZgsqJ0=", "treeSize":"3205307650", "hashes":["tfCB68vh29EGCuUdCAh0c7SkWTkhxdx4BgzG97ksQ6c=", "7mk/WY2lqGuRTbQKWtV6zQnOdC4m3It2CB/bnMVG0T0=", "cCW6sHzYYMHsBtt/nC0rcoi+UvaxxDmIhd7a2M452G4=", "dtWd+u22D+tZsDa84T5ZJW4w8w8UVj9lD0NJJDqjoHc=", "63fJ5uSg1REGSodPGe4zXFCUfdneQIGhUrpJayO0qkA=", "bFh/+uIRRy9QjkgdrzkEoxjpRf5jf0Ls3F1ULwcdbR4=", "M6d/KuQ//DRltSQtXzLZ1oBzSuFsf9G7CmHdK66xTD8=", "u+n9nq8zizYghdQXamNSsJ1yejTkpNxZoMaQxLGJgr8=", "w3goxeMWPeeGsKDqCTPfJBoJKCBga4F5WnerVtyidt8=", "0aSE/O6jCq82vSwoybvpblRnnP72+1gEInKy6Wyc3E0=", "5egUlZ4Pi0Ix/qLyCDXocsWywJCHqrde/S06vOZvlOM=", "QsXufUpoKtSjKAB19K1G00umQpWvPHlBNuRGFiiVjQQ=", "QIMkwQbwAewhLJd//5FFwI3SnyNQBGjCgJrelfG8vhc=", "Et17CACUv9QN64irUJuBjsrgRGkGnnJ9xacRV/aBdwQ=", "lqgEwkxhDMs/gdIAE4lmjfMwg7nDmv7lda6O0LGliCw=", "HovTPBWzoErvUg1u5sIi3x1ZonZ7yJekVsBrmpomCIA=", "G+nk7mgE5h6dUmv2UY7NDJrY3WvrZY31raaO1AETt+o=", "qlTWNDzERnTBQVcHHXXVPjiUP0SsikGxfcxVlgqVFDo=", "gg9GNApNke3+zNhl4tsWtmIycCFrOMCB3lys6XuSQNQ=", "NlOclldmKPA1tMKIskqHAwuQDYHp31YwpK+OOMrMccc=", "qxzHanAzz57SDdmJe0B7bJK72NTIBbwEMGBBKvDOROw=", "xH/DCseLHr9eKoYT8qsORZK7zVdEGYWHuVtsVrD95wY="], "checkpoint":{"envelope":"rekor.sigstore.dev - 1193050959916656506\n3205307650\n0n+xNJnLMgp7sozEH6SRp+aDmxlicUgusBhZlZgsqJ0=\n\n— rekor.sigstore.dev wNI9ajBFAiEA8G/cygCtz/9naiuZybe4zJFaOz4VLpZ1iK0GcNhfBekCIHsqXQhL6AgKCM9O4yJHjKODCbwaSWuacB9SlITFqmLl\n"}}, "canonicalizedBody":"eyJhcGlWZXJzaW9uIjoiMC4wLjEiLCJraW5kIjoiaGFzaGVkcmVrb3JkIiwic3BlYyI6eyJkYXRhIjp7Imhhc2giOnsiYWxnb3JpdGhtIjoic2hhMjU2IiwidmFsdWUiOiI0MDgzOGUyOGE4MjJkMmUzYTA1ZGM4Mjk2N2QzOTMzOWZiNTJlZTNiZjJiNWVkOGFkZWIyMmVjNTMzMzhmYjliIn19LCJzaWduYXR1cmUiOnsiY29udGVudCI6Ik1FUUNJQlE5WlNkbk9jOW5xZnQ4M0dETmE3ODlrZzBISFQ0YWxtUitFUDdoU2lVa0FpQk04NFl1MnJFNFY5NHZOY1pqUEJobWFWK2tBbzhjbFZEZmNlcmtyc1UwclE9PSIsInB1YmxpY0tleSI6eyJjb250ZW50IjoiTFMwdExTMUNSVWRKVGlCRFJWSlVTVVpKUTBGVVJTMHRMUzB0Q2sxSlNVaExla05EUW5KTFowRjNTVUpCWjBsVlRrTndTM2R4YTJvNVNFWTNMek5yZW5KdWFVbE9TMWQ2ZWxWTmQwTm5XVWxMYjFwSmVtb3dSVUYzVFhjS1RucEZWazFDVFVkQk1WVkZRMmhOVFdNeWJHNWpNMUoyWTIxVmRWcEhWakpOVWpSM1NFRlpSRlpSVVVSRmVGWjZZVmRrZW1SSE9YbGFVekZ3WW01U2JBcGpiVEZzV2tkc2FHUkhWWGRJYUdOT1RXcFplRTFFUlRWTmFrbDZUVlJGTVZkb1kwNU5hbGw0VFVSRk5VMXFTVEJOVkVVeFYycEJRVTFHYTNkRmQxbElDa3R2V2tsNmFqQkRRVkZaU1V0dldrbDZhakJFUVZGalJGRm5RVVV6WldSemRYZHVTamRxY1hGbGJVeDRORUZhYkdkU1dEVkllR2hqVjFKRk5rbE9Va1lLZFhWSVltMWFTa05zWm5ndmFYQnFiWE0yYmxKQllWaEZXR05TUVZOWE0yeExjMGx3V1dWVGNpOVVNVXRGWWpBd0x6WlBRMEprUlhkbloxaE9UVUUwUndwQk1WVmtSSGRGUWk5M1VVVkJkMGxJWjBSQlZFSm5UbFpJVTFWRlJFUkJTMEpuWjNKQ1owVkdRbEZqUkVGNlFXWkNaMDVXU0ZOTlJVZEVRVmRuUWxSbUNqQXJibEJXYVZGU2JIWnRiekpQYTI5V1lVeEhUR2hvYTFCNlFuSkNaMDVXU0ZKRlFrRm1PRVZaVkVKbWFHd3hiMlJJVW5kamVtOTJUREprY0dSSGFERUtXV2sxYW1JeU1IWmhTRlp1V2pKc2RWb3lXbWhaTWxWMllUSldlV0p0Vm5OamVURnFZakl4ZEdSWE5YQmtTR3QyVEcxa2NHUkhhREZaYVRrellqTktjZ3BhYlhoMlpETk5kbGx1Vm5CaVIxRjFaVmRHZEdKRlFubGFWMXA2VERKb2JGbFhVbnBNTWpGb1lWYzBkMDlSV1V0TGQxbENRa0ZIUkhaNlFVSkJVVkZ5Q21GSVVqQmpTRTAyVEhrNU1HSXlkR3hpYVRWb1dUTlNjR0l5TlhwTWJXUndaRWRvTVZsdVZucGFXRXBxWWpJMU1GcFhOVEJNYlU1MllsUkJaa0puYjNJS1FtZEZSVUZaVHk5TlFVVkRRa0pHTTJJelNuSmFiWGgyWkRFNWEyRllUbmRaV0ZKcVlVUkJNa0puYjNKQ1owVkZRVmxQTDAxQlJVUkNRMmh0V2xSS2FRcFphbGt5V20xV2JFNXRUWGxaVkdNeldsZE9hbHBxUVRSWlZFRXhUVVJvYTFsNlZUTk9SRWswV2tSUk5FOUhWbXROUWsxSFEybHpSMEZSVVVKbk56aDNDa0ZSVVVWQ1ZVb3hZVmQ0YTAxRGMwZERhWE5IUVZGUlFtYzNPSGRCVVZWRlNGZG9NVm95WkhCaWJXUnRXVmRPYkV3eWRHeGpiVFZzWWtoTmRGa3lPWFFLWWxoV2RXRllValZOUWpCSFEybHpSMEZSVVVKbk56aDNRVkZaUlVRelNteGFiazEyWVVkV2FGcElUWFppVjBad1ltcEJOMEpuYjNKQ1owVkZRVmxQTHdwTlFVVkpRa013VFVzeWFEQmtTRUo2VDJrNGRtUkhPWEphVnpSMVdWZE9NR0ZYT1hWamVUVnVZVmhTYjJSWFNqRmpNbFo1V1RJNWRXUkhWblZrUXpWcUNtSXlNSGRpVVZsTFMzZFpRa0pCUjBSMmVrRkNRMUZTWmtSR01XOWtTRkozWTNwdmRrd3laSEJrUjJneFdXazFhbUl5TUhaaFNGWnVXakpzZFZveVdtZ0tXVEpWZG1FeVZubGliVlp6WTNreGFtSXlNWFJrVnpWd1pFaHJka3h0WkhCa1IyZ3hXV2s1TTJJelNuSmFiWGgyWkROTmRsbHVWbkJpUjFGMVpWZEdkQXBpUlVKNVdsZGFla3d5YUd4WlYxSjZUREl4YUdGWE5IZFBRVmxMUzNkWlFrSkJSMFIyZWtGQ1EyZFJjVVJEYUcxYVZFcHBXV3BaTWxwdFZteE9iVTE1Q2xsVVl6TmFWMDVxV21wQk5GbFVRVEZOUkdocldYcFZNMDVFU1RSYVJGRTBUMGRXYTAxQ2MwZERhWE5IUVZGUlFtYzNPSGRCVVhORlJGRjNUR015Vm5NS1dta3hiMkl6VGpCYVYxRjNVVUZaUzB0M1dVSkNRVWRFZG5wQlFrUkJVWGxFUkVKdlpFaFNkMk42YjNaTU1tUndaRWRvTVZscE5XcGlNakIyWVVoV2JncGFNbXgxV2pKYWFGa3lWWFpoTWxaNVltMVdjMk41TVdwaU1qRjBaRmMxY0dSSWEzZFBRVmxMUzNkWlFrSkJSMFIyZWtGQ1JGRlJjVVJEYUcxYVZFcHBDbGxxV1RKYWJWWnNUbTFOZVZsVVl6TmFWMDVxV21wQk5GbFVRVEZOUkdocldYcFZNMDVFU1RSYVJGRTBUMGRXYTAxQ09FZERhWE5IUVZGUlFtYzNPSGNLUVZFMFJVVlJkMUJqYlZadFkzazViMXBYUm10amVUbDBXVmRzZFUxQ2IwZERhWE5IUVZGUlFtYzNPSGRCVVRoRlJFRjNTMDFVUVROTlZGRXpUbFJWZVFwUFZFRjFRbWR2Y2tKblJVVkJXVTh2VFVGRlVVSkRRVTFJYldnd1pFaENlazlwT0haYU1td3dZVWhXYVV4dFRuWmlVemx2WkZka2JtRlhOVzVhYlVacUNscFVRVmxDWjI5eVFtZEZSVUZaVHk5TlFVVlNRa0Z2VFVORVNURk9la2wzVG5wUmVrMUhNRWREYVhOSFFWRlJRbWMzT0hkQlVrbEZXSGQ0WkdGSVVqQUtZMGhOTmt4NU9XNWhXRkp2WkZkSmRWa3lPWFJNTW1neFdqSmtjR0p0WkcxWlYwNXNUREowYkdOdE5XeGlTRTEwV1RJNWRHSllWblZoV0ZJMVRIazFiZ3BoV0ZKdlpGZEpkbVF5T1hsaE1scHpZak5rZWt3eVNqRmhWM2hyVEc1c2FHSlhlRUZqYlZadFkzazViMXBYUm10amVUbDBXVmRzZFUxRVowZERhWE5IQ2tGUlVVSm5OemgzUVZKTlJVdG5kMjlhYlZWNVdXMUpNazV0V214YVZGcHFUVzFGTTA0eVZtcFpNbGwzVDBkRmQwNVVRVFJhUjAweFRucFJlVTlIVVRBS1QwUm9iRnBFUVdoQ1oyOXlRbWRGUlVGWlR5OU5RVVZWUWtKTlRVVllaSFpqYlhSdFlrYzVNMWd5VW5Cak0wSm9aRWRPYjAxSFVVZERhWE5IUVZGUlFncG5OemgzUVZKVlJWWm5lRlZoU0ZJd1kwaE5Oa3g1T1c1aFdGSnZaRmRKZFZreU9YUk1NbWd4V2pKa2NHSnRaRzFaVjA1c1RESjBiR050Tld4aVNFMTBDbGt5T1hSaVdGWjFZVmhTTlV3eVJtcGtSMngyWW01TmRtTnVWblZqZVRoNlQxUkplazlVVlhoUFJFVjNUa001YUdSSVVteGlXRUl3WTNrNGVFMUNXVWNLUTJselIwRlJVVUpuTnpoM1FWSlpSVU5CZDBkalNGWnBZa2RzYWsxRldVZERhWE5IUVZGUlFtYzNPSGRCVW1kRlQwRjNNbU50Vm5kaWVuQnZaRmRrYmdwaFZ6VnVXbTFHYWxwVE9YSmFXRXAxV2xkNGVreFhUblppVnpFeFltMXNNR1ZVY0hsYVYxazJZMjFXYldONU9XOWFWMFpyWTNrNWRGbFhiSFZOU1VkTENrSm5iM0pDWjBWRlFXUmFOVUZuVVVOQ1NIZEZaV2RDTkVGSVdVRXpWREIzWVhOaVNFVlVTbXBIVWpSamJWZGpNMEZ4U2t0WWNtcGxVRXN6TDJnMGNIa0taME00Y0Rkdk5FRkJRVWRvVm10d1lUWkJRVUZDUVUxQlVucENSa0ZwUlVGd1VYZERRMWc1Tm1KcGJXTlRObmhTU2t0WFNrSnpWV2xUYjBSMlpDdFFhd292VVZsWlJtTnJkRk5XYTBOSlJVbzVMekp5UjFvNGEyeFljMng1ZEZaclpVdEdVWHB5SzJVMGJEZE9iMjkwVkRKRWRUaDFWbWhoWTAxQmIwZERRM0ZIQ2xOTk5EbENRVTFFUVRKalFVMUhVVU5OUWtGRlQyMXhaR2t3UkRoU01sWXhjVTFuTkM5UGVYSjJWRXBYZFRSdllXd3dTMFZsWmxkc2RUWkhSamR6YjJVS1VGTklhMEV3TjFwNVJXbHpkR1UwUWtSblNYZG1OV1JYVG5BM2RtbFFaMFJPVTJaMVluVTJTSE42Y2trdk5FWm5ZVTF3TldoUVFVaG1OV1l2TDFWQk5RcHVUV0puUlU1dVdWWjJlRmRHUzBKRFZsbzRhZ290TFMwdExVVk9SQ0JEUlZKVVNVWkpRMEZVUlMwdExTMHRDZz09In19fX0="}], "timestampVerificationData":{"rfc3161Timestamps":[{"signedTimestamp":"MIICyzADAgEAMIICwgYJKoZIhvcNAQcCoIICszCCAq8CAQMxDTALBglghkgBZQMEAgEwgbgGCyqGSIb3DQEJEAEEoIGoBIGlMIGiAgEBBgkrBgEEAYO/MAIwMTANBglghkgBZQMEAgEFAAQgo8yY2H2W5TJzDMi71ksxVvtMI5Yk85L5ePrKyhWWYuMCFQCmdZ6tqY2ljRgQDYyeRbpJ+UjT0hgPMjAyNjEwMTkyMjMxMTZaMAMCAQGgMqQwMC4xFTATBgNVBAoTDHNpZ3N0b3JlLmRldjEVMBMGA1UEAxMMc2lnc3RvcmUtdHNhoAAxggHcMIIB2AIBATBRMDkxFTATBgNVBAoTDHNpZ3N0b3JlLmRldjEgMB4GA1UEAxMXc2lnc3RvcmUtdHNhLXNlbGZzaWduZWQCFDoTVC8MkGHuvMFDL8uKjosqI4sMMAsGCWCGSAFlAwQCAaCB/DAaBgkqhkiG9w0BCQMxDQYLKoZIhvcNAQkQAQQwHAYJKoZIhvcNAQkFMQ8XDTI2MTAxOTIyMzExNlowLwYJKoZIhvcNAQkEMSIEIOsIp7PBFnHyxWzK5qw8TodpDGtaKM4KYOySCt5HM4kaMIGOBgsqhkiG9w0BCRACLzF/MH0wezB5BCCF+Se8B6tiysO0Q1bBDvyBssaIP9p6uebYcNnROs0FtzBVMD2kOzA5MRUwEwYDVQQKEwxzaWdzdG9yZS5kZXYxIDAeBgNVBAMTF3NpZ3N0b3JlLXRzYS1zZWxmc2lnbmVkAhQ6E1QvDJBh7rzBQy/Lio6LKiOLDDAKBggqhkjOPQQDAgRoMGYCMQD799LG2rm5BHbJ+kOEhatzZmFa2atv0WZ5e8hV1h6NdYDUIQYkTYLtLfWUt9FdwEUCMQDd0xGZB1eu/B0kZhmEIO14nniSqN90/Zrh4CHX25WEB+okXRo3Ug41mIWRZeMv0bM="}]}}, "messageSignature":{"messageDigest":{"algorithm":"SHA2_256", "digest":"QIOOKKgi0uOgXcgpZ9OTOftS7jvyte2K3rIuxTM4+5s="}, "signature":"MEQCIBQ9ZSdnOc9nqft83GDNa789kg0HHT4almR+EP7hSiUkAiBM84Yu2rE4V94vNcZjPBhmaV+kAo8clVDfcerkrsU0rQ=="}}
|
build/torch212-rocm71-x86_64-linux/mha.py
ADDED
|
@@ -0,0 +1,332 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""FlashAttention forward entry points backed by aiter's Composable-Kernel FMHA.
|
| 2 |
+
|
| 3 |
+
This is the compiled (HIP / Composable Kernel) counterpart to the Triton
|
| 4 |
+
``aiter-flash-attn`` package. Only the *forward* pass is provided (dense and
|
| 5 |
+
variable-length), including support for learnable attention sinks via
|
| 6 |
+
``window_size[2]`` / ``s_aux`` (e.g. gpt-oss). The op set ships exclusively
|
| 7 |
+
the CK ``mha_fwd`` / ``mha_varlen_fwd`` kernels, so the sink path is always
|
| 8 |
+
taken (no fall-through to a non-sink ASM kernel).
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
from typing import Optional, Tuple, Union
|
| 12 |
+
|
| 13 |
+
import torch
|
| 14 |
+
from torch import Tensor
|
| 15 |
+
|
| 16 |
+
from ._ops import ops
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def _round_multiple(x: int, m: int) -> int:
|
| 20 |
+
return (x + m - 1) // m * m
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def _maybe_pad_headdim(t: Tensor, target: int) -> Tensor:
|
| 24 |
+
og = t.size(-1)
|
| 25 |
+
if og == target:
|
| 26 |
+
return t
|
| 27 |
+
return torch.nn.functional.pad(t, [0, target - og])
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def _unpack_window(window_size) -> Tuple[int, int, int]:
|
| 31 |
+
left = int(window_size[0])
|
| 32 |
+
right = int(window_size[1])
|
| 33 |
+
sink = int(window_size[2]) if len(window_size) == 3 else 0
|
| 34 |
+
return left, right, sink
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def _prep_sink(s_aux: Optional[Tensor], nheads: int, device) -> Optional[Tensor]:
|
| 38 |
+
if s_aux is None:
|
| 39 |
+
return None
|
| 40 |
+
assert s_aux.device == device, "s_aux must be on the same device as q"
|
| 41 |
+
assert s_aux.shape[0] == nheads, "s_aux must have shape [nheads_q]"
|
| 42 |
+
if s_aux.dtype != torch.float32:
|
| 43 |
+
s_aux = s_aux.to(torch.float32)
|
| 44 |
+
return s_aux
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def _effective_sink_size(sink_size: int, s_aux: Optional[Tensor]) -> int:
|
| 48 |
+
# The CK kernel only takes the sink path when sink_size > 0. Callers using
|
| 49 |
+
# the transformers attention interface pass the per-head sink logits as
|
| 50 |
+
# ``s_aux`` but leave ``window_size`` a 2-tuple (sink_size == 0), so enable
|
| 51 |
+
# a single learnable sink automatically when s_aux is given.
|
| 52 |
+
if s_aux is not None and sink_size == 0:
|
| 53 |
+
return 1
|
| 54 |
+
return sink_size
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def flash_attn_func(
|
| 58 |
+
q: Tensor,
|
| 59 |
+
k: Tensor,
|
| 60 |
+
v: Tensor,
|
| 61 |
+
dropout_p: float = 0.0,
|
| 62 |
+
softmax_scale: Optional[float] = None,
|
| 63 |
+
causal: bool = False,
|
| 64 |
+
window_size: Tuple[int, ...] = (-1, -1, 0),
|
| 65 |
+
bias: Optional[Tensor] = None,
|
| 66 |
+
alibi_slopes: Optional[Tensor] = None,
|
| 67 |
+
return_lse: bool = False,
|
| 68 |
+
return_attn_probs: bool = False,
|
| 69 |
+
s_aux: Optional[Tensor] = None,
|
| 70 |
+
**kwargs,
|
| 71 |
+
) -> Union[Tensor, Tuple[Tensor, ...]]:
|
| 72 |
+
"""Dense FlashAttention forward.
|
| 73 |
+
|
| 74 |
+
``q, k, v`` have shape ``(batch, seqlen, nheads, headdim)``. ``window_size``
|
| 75 |
+
is ``(left, right, sink_size)``; ``s_aux`` is an optional ``(nheads_q,)``
|
| 76 |
+
fp32 tensor of learnable sink logits. Extra keyword arguments (e.g. those
|
| 77 |
+
passed by the transformers flash-attention interface) are ignored.
|
| 78 |
+
"""
|
| 79 |
+
if softmax_scale is None:
|
| 80 |
+
softmax_scale = q.shape[-1] ** (-0.5)
|
| 81 |
+
window_size_left, window_size_right, sink_size = _unpack_window(window_size)
|
| 82 |
+
|
| 83 |
+
nheads_q = q.size(2)
|
| 84 |
+
s_aux = _prep_sink(s_aux, nheads_q, q.device)
|
| 85 |
+
sink_size = _effective_sink_size(sink_size, s_aux)
|
| 86 |
+
|
| 87 |
+
head_q_og = q.size(-1)
|
| 88 |
+
head_v_og = v.size(-1)
|
| 89 |
+
head_q = _round_multiple(head_q_og, 8)
|
| 90 |
+
head_v = _round_multiple(head_v_og, 8)
|
| 91 |
+
q = _maybe_pad_headdim(q, head_q)
|
| 92 |
+
k = _maybe_pad_headdim(k, head_q)
|
| 93 |
+
v = _maybe_pad_headdim(v, head_v)
|
| 94 |
+
|
| 95 |
+
return_softmax = return_attn_probs and dropout_p > 0
|
| 96 |
+
out, softmax_lse, S_dmask, _ = ops.mha_fwd(
|
| 97 |
+
q,
|
| 98 |
+
k,
|
| 99 |
+
v,
|
| 100 |
+
dropout_p,
|
| 101 |
+
softmax_scale,
|
| 102 |
+
causal,
|
| 103 |
+
window_size_left,
|
| 104 |
+
window_size_right,
|
| 105 |
+
sink_size,
|
| 106 |
+
return_lse or return_attn_probs,
|
| 107 |
+
return_softmax,
|
| 108 |
+
None, # cu_seqlens_q
|
| 109 |
+
None, # cu_seqlens_kv
|
| 110 |
+
None, # out
|
| 111 |
+
bias,
|
| 112 |
+
alibi_slopes,
|
| 113 |
+
None, # q_descale
|
| 114 |
+
None, # k_descale
|
| 115 |
+
None, # v_descale
|
| 116 |
+
s_aux,
|
| 117 |
+
None, # gen
|
| 118 |
+
)
|
| 119 |
+
|
| 120 |
+
out = out[..., :head_v_og]
|
| 121 |
+
if not (return_lse or return_attn_probs):
|
| 122 |
+
return out
|
| 123 |
+
result = (out,)
|
| 124 |
+
if return_lse:
|
| 125 |
+
result = result + (softmax_lse,)
|
| 126 |
+
if return_attn_probs:
|
| 127 |
+
result = result + (S_dmask,)
|
| 128 |
+
return result
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
def flash_attn_varlen_func(
|
| 132 |
+
q: Tensor,
|
| 133 |
+
k: Tensor,
|
| 134 |
+
v: Tensor,
|
| 135 |
+
cu_seqlens_q: Tensor,
|
| 136 |
+
cu_seqlens_k: Tensor,
|
| 137 |
+
max_seqlen_q: int,
|
| 138 |
+
max_seqlen_k: int,
|
| 139 |
+
dropout_p: float = 0.0,
|
| 140 |
+
softmax_scale: Optional[float] = None,
|
| 141 |
+
causal: bool = False,
|
| 142 |
+
window_size: Tuple[int, ...] = (-1, -1, 0),
|
| 143 |
+
bias: Optional[Tensor] = None,
|
| 144 |
+
alibi_slopes: Optional[Tensor] = None,
|
| 145 |
+
return_lse: bool = False,
|
| 146 |
+
return_attn_probs: bool = False,
|
| 147 |
+
block_table: Optional[Tensor] = None,
|
| 148 |
+
min_seqlen_q: int = 0,
|
| 149 |
+
logits_soft_cap: float = 0.0,
|
| 150 |
+
zero_tensors: bool = False,
|
| 151 |
+
cu_seqlens_q_padded: Optional[Tensor] = None,
|
| 152 |
+
cu_seqlens_k_padded: Optional[Tensor] = None,
|
| 153 |
+
s_aux: Optional[Tensor] = None,
|
| 154 |
+
**kwargs,
|
| 155 |
+
) -> Union[Tensor, Tuple[Tensor, ...]]:
|
| 156 |
+
"""Variable-length / packed FlashAttention forward.
|
| 157 |
+
|
| 158 |
+
``q`` has shape ``(total_q, nheads, headdim)``; ``cu_seqlens_*`` are int32
|
| 159 |
+
cumulative offsets of shape ``(batch + 1,)``. Sinks are supported through
|
| 160 |
+
``window_size[2]`` / ``s_aux`` as in :func:`flash_attn_func`. Extra keyword
|
| 161 |
+
arguments (e.g. from the transformers flash-attention interface) are ignored.
|
| 162 |
+
"""
|
| 163 |
+
if softmax_scale is None:
|
| 164 |
+
softmax_scale = q.shape[-1] ** (-0.5)
|
| 165 |
+
window_size_left, window_size_right, sink_size = _unpack_window(window_size)
|
| 166 |
+
|
| 167 |
+
nheads_q = q.size(1)
|
| 168 |
+
s_aux = _prep_sink(s_aux, nheads_q, q.device)
|
| 169 |
+
sink_size = _effective_sink_size(sink_size, s_aux)
|
| 170 |
+
|
| 171 |
+
head_q_og = q.size(-1)
|
| 172 |
+
head_v_og = v.size(-1)
|
| 173 |
+
head_q = _round_multiple(head_q_og, 8)
|
| 174 |
+
head_v = _round_multiple(head_v_og, 8)
|
| 175 |
+
q = _maybe_pad_headdim(q, head_q)
|
| 176 |
+
k = _maybe_pad_headdim(k, head_q)
|
| 177 |
+
v = _maybe_pad_headdim(v, head_v)
|
| 178 |
+
|
| 179 |
+
return_softmax = return_attn_probs and dropout_p > 0
|
| 180 |
+
out, softmax_lse, S_dmask, _ = ops.mha_varlen_fwd(
|
| 181 |
+
q,
|
| 182 |
+
k,
|
| 183 |
+
v,
|
| 184 |
+
cu_seqlens_q,
|
| 185 |
+
cu_seqlens_k,
|
| 186 |
+
max_seqlen_q,
|
| 187 |
+
max_seqlen_k,
|
| 188 |
+
min_seqlen_q,
|
| 189 |
+
dropout_p,
|
| 190 |
+
softmax_scale,
|
| 191 |
+
logits_soft_cap,
|
| 192 |
+
zero_tensors,
|
| 193 |
+
causal,
|
| 194 |
+
window_size_left,
|
| 195 |
+
window_size_right,
|
| 196 |
+
sink_size,
|
| 197 |
+
return_lse or return_attn_probs,
|
| 198 |
+
return_softmax,
|
| 199 |
+
None, # out
|
| 200 |
+
block_table,
|
| 201 |
+
bias,
|
| 202 |
+
alibi_slopes,
|
| 203 |
+
None, # q_descale
|
| 204 |
+
None, # k_descale
|
| 205 |
+
None, # v_descale
|
| 206 |
+
None, # gen
|
| 207 |
+
cu_seqlens_q_padded,
|
| 208 |
+
cu_seqlens_k_padded,
|
| 209 |
+
s_aux,
|
| 210 |
+
)
|
| 211 |
+
|
| 212 |
+
out = out[..., :head_v_og]
|
| 213 |
+
if not (return_lse or return_attn_probs):
|
| 214 |
+
return out
|
| 215 |
+
result = (out,)
|
| 216 |
+
if return_lse:
|
| 217 |
+
result = result + (softmax_lse,)
|
| 218 |
+
if return_attn_probs:
|
| 219 |
+
result = result + (S_dmask,)
|
| 220 |
+
return result
|
| 221 |
+
|
| 222 |
+
|
| 223 |
+
def _append_to_contiguous_cache(k_cache, v_cache, k, v, cache_seqlens, Sq):
|
| 224 |
+
"""In-place append of new k/v into a contiguous cache at per-batch offsets."""
|
| 225 |
+
B = k_cache.shape[0]
|
| 226 |
+
if isinstance(cache_seqlens, int):
|
| 227 |
+
cache_seqlens = torch.full((B,), cache_seqlens, device=k_cache.device, dtype=torch.long)
|
| 228 |
+
pos = cache_seqlens.view(B, 1).to(torch.long) + torch.arange(Sq, device=k_cache.device).view(1, Sq)
|
| 229 |
+
bidx = torch.arange(B, device=k_cache.device).view(B, 1).expand(B, Sq)
|
| 230 |
+
k_cache[bidx, pos] = k.to(k_cache.dtype)
|
| 231 |
+
v_cache[bidx, pos] = v.to(v_cache.dtype)
|
| 232 |
+
|
| 233 |
+
|
| 234 |
+
def flash_attn_with_kvcache(
|
| 235 |
+
q: Tensor,
|
| 236 |
+
k_cache: Tensor,
|
| 237 |
+
v_cache: Tensor,
|
| 238 |
+
k: Optional[Tensor] = None,
|
| 239 |
+
v: Optional[Tensor] = None,
|
| 240 |
+
rotary_cos: Optional[Tensor] = None,
|
| 241 |
+
rotary_sin: Optional[Tensor] = None,
|
| 242 |
+
cache_seqlens=None,
|
| 243 |
+
block_table: Optional[Tensor] = None,
|
| 244 |
+
softmax_scale: Optional[float] = None,
|
| 245 |
+
causal: bool = True,
|
| 246 |
+
window_size: Tuple[int, ...] = (-1, -1, 0),
|
| 247 |
+
s_aux: Optional[Tensor] = None,
|
| 248 |
+
return_lse: bool = False,
|
| 249 |
+
**kwargs,
|
| 250 |
+
) -> Union[Tensor, Tuple[Tensor, ...]]:
|
| 251 |
+
"""FlashAttention against a KV cache, built on the CK split-KV kernel.
|
| 252 |
+
|
| 253 |
+
``q`` is ``(batch, seqlen_q, nheads, headdim)``. For a contiguous cache,
|
| 254 |
+
``k_cache``/``v_cache`` are ``(batch, seqlen_cache, nheads_k, headdim)`` and
|
| 255 |
+
``cache_seqlens`` (int or ``(batch,)``) is the number of valid cache entries
|
| 256 |
+
*before* appending. If ``k``/``v`` (the new tokens) are given, they are
|
| 257 |
+
written into the cache in place. The query attends to the valid prefix
|
| 258 |
+
(``cache_seqlens`` + ``seqlen_q``) with bottom-right causal alignment.
|
| 259 |
+
|
| 260 |
+
Notes / current limitations:
|
| 261 |
+
- Rotary embedding is not applied here; pass ``q``/``k`` already rotated
|
| 262 |
+
(``rotary_cos``/``rotary_sin`` are rejected).
|
| 263 |
+
- Paged caches are addressed via ``block_table`` (forwarded to the split-KV
|
| 264 |
+
varlen kernel); in-cache append for paged layouts must be done by the
|
| 265 |
+
caller (pass ``k=v=None``).
|
| 266 |
+
"""
|
| 267 |
+
if rotary_cos is not None or rotary_sin is not None:
|
| 268 |
+
raise NotImplementedError(
|
| 269 |
+
"flash_attn_with_kvcache does not apply rotary; rotate q/k beforehand."
|
| 270 |
+
)
|
| 271 |
+
if softmax_scale is None:
|
| 272 |
+
softmax_scale = q.shape[-1] ** (-0.5)
|
| 273 |
+
window_size_left, window_size_right, sink_size = _unpack_window(window_size)
|
| 274 |
+
B, Sq, Hq, D = q.shape
|
| 275 |
+
s_aux = _prep_sink(s_aux, Hq, q.device)
|
| 276 |
+
sink_size = _effective_sink_size(sink_size, s_aux)
|
| 277 |
+
|
| 278 |
+
paged = block_table is not None
|
| 279 |
+
|
| 280 |
+
# ---- append new tokens into the cache (contiguous layout only) ----
|
| 281 |
+
if k is not None and not paged:
|
| 282 |
+
assert cache_seqlens is not None, "cache_seqlens required to append into the cache"
|
| 283 |
+
_append_to_contiguous_cache(k_cache, v_cache, k, v, cache_seqlens, Sq)
|
| 284 |
+
elif k is not None and paged:
|
| 285 |
+
raise NotImplementedError(
|
| 286 |
+
"in-cache append for paged layouts is not supported; append externally "
|
| 287 |
+
"and call with k=v=None."
|
| 288 |
+
)
|
| 289 |
+
|
| 290 |
+
appended = 0 if k is None else Sq
|
| 291 |
+
|
| 292 |
+
# ---- key lengths ----
|
| 293 |
+
if cache_seqlens is None:
|
| 294 |
+
klens = torch.full((B,), k_cache.shape[1], device=q.device, dtype=torch.long)
|
| 295 |
+
elif isinstance(cache_seqlens, int):
|
| 296 |
+
klens = torch.full((B,), cache_seqlens + appended, device=q.device, dtype=torch.long)
|
| 297 |
+
else:
|
| 298 |
+
klens = cache_seqlens.to(q.device, torch.long) + appended
|
| 299 |
+
|
| 300 |
+
uniform = bool((klens == klens[0]).all().item())
|
| 301 |
+
|
| 302 |
+
# ---- fast path: contiguous cache, uniform length -> dense kernel ----
|
| 303 |
+
if not paged and uniform:
|
| 304 |
+
L = int(klens[0].item())
|
| 305 |
+
return flash_attn_func(
|
| 306 |
+
q, k_cache[:, :L], v_cache[:, :L],
|
| 307 |
+
softmax_scale=softmax_scale, causal=causal,
|
| 308 |
+
window_size=(window_size_left, window_size_right, sink_size),
|
| 309 |
+
s_aux=s_aux, return_lse=return_lse,
|
| 310 |
+
)
|
| 311 |
+
|
| 312 |
+
# ---- general path: variable cache lengths -> split-KV varlen kernel ----
|
| 313 |
+
cu_q = torch.arange(0, (B + 1) * Sq, Sq, device=q.device, dtype=torch.int32)
|
| 314 |
+
cu_k = torch.zeros(B + 1, device=q.device, dtype=torch.int32)
|
| 315 |
+
cu_k[1:] = torch.cumsum(klens, 0).to(torch.int32)
|
| 316 |
+
q_packed = q.reshape(B * Sq, Hq, D)
|
| 317 |
+
if paged:
|
| 318 |
+
k_packed, v_packed = k_cache, v_cache # kernel gathers via block_table
|
| 319 |
+
else:
|
| 320 |
+
k_packed = torch.cat([k_cache[b, : int(klens[b])] for b in range(B)], dim=0)
|
| 321 |
+
v_packed = torch.cat([v_cache[b, : int(klens[b])] for b in range(B)], dim=0)
|
| 322 |
+
|
| 323 |
+
out = flash_attn_varlen_func(
|
| 324 |
+
q_packed, k_packed, v_packed, cu_q, cu_k, Sq, int(klens.max().item()),
|
| 325 |
+
softmax_scale=softmax_scale, causal=causal,
|
| 326 |
+
window_size=(window_size_left, window_size_right, sink_size),
|
| 327 |
+
block_table=block_table, s_aux=s_aux, return_lse=return_lse,
|
| 328 |
+
)
|
| 329 |
+
if return_lse:
|
| 330 |
+
out, lse = out
|
| 331 |
+
return out.reshape(B, Sq, Hq, D), lse
|
| 332 |
+
return out.reshape(B, Sq, Hq, D)
|
build/torch212-rocm71-x86_64-linux/symbols.json
ADDED
|
@@ -0,0 +1,349 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": 1,
|
| 3 |
+
"module": "_aiter_flash_attn_ck_rocm_fe2bb66",
|
| 4 |
+
"functions": [
|
| 5 |
+
{
|
| 6 |
+
"name": "flash_attn_func",
|
| 7 |
+
"qualname": "flash_attn_func",
|
| 8 |
+
"module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha",
|
| 9 |
+
"kind": "function",
|
| 10 |
+
"doc": "Dense FlashAttention forward.\n\n``q, k, v`` have shape ``(batch, seqlen, nheads, headdim)``. ``window_size``\nis ``(left, right, sink_size)``; ``s_aux`` is an optional ``(nheads_q,)``\nfp32 tensor of learnable sink logits. Extra keyword arguments (e.g. those\npassed by the transformers flash-attention interface) are ignored.",
|
| 11 |
+
"signature": {
|
| 12 |
+
"parameters": [
|
| 13 |
+
{
|
| 14 |
+
"name": "q",
|
| 15 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 16 |
+
"default": null,
|
| 17 |
+
"annotation": "torch.Tensor"
|
| 18 |
+
},
|
| 19 |
+
{
|
| 20 |
+
"name": "k",
|
| 21 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 22 |
+
"default": null,
|
| 23 |
+
"annotation": "torch.Tensor"
|
| 24 |
+
},
|
| 25 |
+
{
|
| 26 |
+
"name": "v",
|
| 27 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 28 |
+
"default": null,
|
| 29 |
+
"annotation": "torch.Tensor"
|
| 30 |
+
},
|
| 31 |
+
{
|
| 32 |
+
"name": "dropout_p",
|
| 33 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 34 |
+
"default": "0.0",
|
| 35 |
+
"annotation": "float"
|
| 36 |
+
},
|
| 37 |
+
{
|
| 38 |
+
"name": "softmax_scale",
|
| 39 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 40 |
+
"default": "None",
|
| 41 |
+
"annotation": "float | None"
|
| 42 |
+
},
|
| 43 |
+
{
|
| 44 |
+
"name": "causal",
|
| 45 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 46 |
+
"default": "False",
|
| 47 |
+
"annotation": "bool"
|
| 48 |
+
},
|
| 49 |
+
{
|
| 50 |
+
"name": "window_size",
|
| 51 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 52 |
+
"default": "(-1, -1, 0)",
|
| 53 |
+
"annotation": "Tuple[int, ...]"
|
| 54 |
+
},
|
| 55 |
+
{
|
| 56 |
+
"name": "bias",
|
| 57 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 58 |
+
"default": "None",
|
| 59 |
+
"annotation": "torch.Tensor | None"
|
| 60 |
+
},
|
| 61 |
+
{
|
| 62 |
+
"name": "alibi_slopes",
|
| 63 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 64 |
+
"default": "None",
|
| 65 |
+
"annotation": "torch.Tensor | None"
|
| 66 |
+
},
|
| 67 |
+
{
|
| 68 |
+
"name": "return_lse",
|
| 69 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 70 |
+
"default": "False",
|
| 71 |
+
"annotation": "bool"
|
| 72 |
+
},
|
| 73 |
+
{
|
| 74 |
+
"name": "return_attn_probs",
|
| 75 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 76 |
+
"default": "False",
|
| 77 |
+
"annotation": "bool"
|
| 78 |
+
},
|
| 79 |
+
{
|
| 80 |
+
"name": "s_aux",
|
| 81 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 82 |
+
"default": "None",
|
| 83 |
+
"annotation": "torch.Tensor | None"
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"name": "kwargs",
|
| 87 |
+
"kind": "VAR_KEYWORD",
|
| 88 |
+
"default": null,
|
| 89 |
+
"annotation": null
|
| 90 |
+
}
|
| 91 |
+
],
|
| 92 |
+
"return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]"
|
| 93 |
+
}
|
| 94 |
+
},
|
| 95 |
+
{
|
| 96 |
+
"name": "flash_attn_varlen_func",
|
| 97 |
+
"qualname": "flash_attn_varlen_func",
|
| 98 |
+
"module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha",
|
| 99 |
+
"kind": "function",
|
| 100 |
+
"doc": "Variable-length / packed FlashAttention forward.\n\n``q`` has shape ``(total_q, nheads, headdim)``; ``cu_seqlens_*`` are int32\ncumulative offsets of shape ``(batch + 1,)``. Sinks are supported through\n``window_size[2]`` / ``s_aux`` as in :func:`flash_attn_func`. Extra keyword\narguments (e.g. from the transformers flash-attention interface) are ignored.",
|
| 101 |
+
"signature": {
|
| 102 |
+
"parameters": [
|
| 103 |
+
{
|
| 104 |
+
"name": "q",
|
| 105 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 106 |
+
"default": null,
|
| 107 |
+
"annotation": "torch.Tensor"
|
| 108 |
+
},
|
| 109 |
+
{
|
| 110 |
+
"name": "k",
|
| 111 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 112 |
+
"default": null,
|
| 113 |
+
"annotation": "torch.Tensor"
|
| 114 |
+
},
|
| 115 |
+
{
|
| 116 |
+
"name": "v",
|
| 117 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 118 |
+
"default": null,
|
| 119 |
+
"annotation": "torch.Tensor"
|
| 120 |
+
},
|
| 121 |
+
{
|
| 122 |
+
"name": "cu_seqlens_q",
|
| 123 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 124 |
+
"default": null,
|
| 125 |
+
"annotation": "torch.Tensor"
|
| 126 |
+
},
|
| 127 |
+
{
|
| 128 |
+
"name": "cu_seqlens_k",
|
| 129 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 130 |
+
"default": null,
|
| 131 |
+
"annotation": "torch.Tensor"
|
| 132 |
+
},
|
| 133 |
+
{
|
| 134 |
+
"name": "max_seqlen_q",
|
| 135 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 136 |
+
"default": null,
|
| 137 |
+
"annotation": "int"
|
| 138 |
+
},
|
| 139 |
+
{
|
| 140 |
+
"name": "max_seqlen_k",
|
| 141 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 142 |
+
"default": null,
|
| 143 |
+
"annotation": "int"
|
| 144 |
+
},
|
| 145 |
+
{
|
| 146 |
+
"name": "dropout_p",
|
| 147 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 148 |
+
"default": "0.0",
|
| 149 |
+
"annotation": "float"
|
| 150 |
+
},
|
| 151 |
+
{
|
| 152 |
+
"name": "softmax_scale",
|
| 153 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 154 |
+
"default": "None",
|
| 155 |
+
"annotation": "float | None"
|
| 156 |
+
},
|
| 157 |
+
{
|
| 158 |
+
"name": "causal",
|
| 159 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 160 |
+
"default": "False",
|
| 161 |
+
"annotation": "bool"
|
| 162 |
+
},
|
| 163 |
+
{
|
| 164 |
+
"name": "window_size",
|
| 165 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 166 |
+
"default": "(-1, -1, 0)",
|
| 167 |
+
"annotation": "Tuple[int, ...]"
|
| 168 |
+
},
|
| 169 |
+
{
|
| 170 |
+
"name": "bias",
|
| 171 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 172 |
+
"default": "None",
|
| 173 |
+
"annotation": "torch.Tensor | None"
|
| 174 |
+
},
|
| 175 |
+
{
|
| 176 |
+
"name": "alibi_slopes",
|
| 177 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 178 |
+
"default": "None",
|
| 179 |
+
"annotation": "torch.Tensor | None"
|
| 180 |
+
},
|
| 181 |
+
{
|
| 182 |
+
"name": "return_lse",
|
| 183 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 184 |
+
"default": "False",
|
| 185 |
+
"annotation": "bool"
|
| 186 |
+
},
|
| 187 |
+
{
|
| 188 |
+
"name": "return_attn_probs",
|
| 189 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 190 |
+
"default": "False",
|
| 191 |
+
"annotation": "bool"
|
| 192 |
+
},
|
| 193 |
+
{
|
| 194 |
+
"name": "block_table",
|
| 195 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 196 |
+
"default": "None",
|
| 197 |
+
"annotation": "torch.Tensor | None"
|
| 198 |
+
},
|
| 199 |
+
{
|
| 200 |
+
"name": "min_seqlen_q",
|
| 201 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 202 |
+
"default": "0",
|
| 203 |
+
"annotation": "int"
|
| 204 |
+
},
|
| 205 |
+
{
|
| 206 |
+
"name": "logits_soft_cap",
|
| 207 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 208 |
+
"default": "0.0",
|
| 209 |
+
"annotation": "float"
|
| 210 |
+
},
|
| 211 |
+
{
|
| 212 |
+
"name": "zero_tensors",
|
| 213 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 214 |
+
"default": "False",
|
| 215 |
+
"annotation": "bool"
|
| 216 |
+
},
|
| 217 |
+
{
|
| 218 |
+
"name": "cu_seqlens_q_padded",
|
| 219 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 220 |
+
"default": "None",
|
| 221 |
+
"annotation": "torch.Tensor | None"
|
| 222 |
+
},
|
| 223 |
+
{
|
| 224 |
+
"name": "cu_seqlens_k_padded",
|
| 225 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 226 |
+
"default": "None",
|
| 227 |
+
"annotation": "torch.Tensor | None"
|
| 228 |
+
},
|
| 229 |
+
{
|
| 230 |
+
"name": "s_aux",
|
| 231 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 232 |
+
"default": "None",
|
| 233 |
+
"annotation": "torch.Tensor | None"
|
| 234 |
+
},
|
| 235 |
+
{
|
| 236 |
+
"name": "kwargs",
|
| 237 |
+
"kind": "VAR_KEYWORD",
|
| 238 |
+
"default": null,
|
| 239 |
+
"annotation": null
|
| 240 |
+
}
|
| 241 |
+
],
|
| 242 |
+
"return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]"
|
| 243 |
+
}
|
| 244 |
+
},
|
| 245 |
+
{
|
| 246 |
+
"name": "flash_attn_with_kvcache",
|
| 247 |
+
"qualname": "flash_attn_with_kvcache",
|
| 248 |
+
"module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha",
|
| 249 |
+
"kind": "function",
|
| 250 |
+
"doc": "FlashAttention against a KV cache, built on the CK split-KV kernel.\n\n``q`` is ``(batch, seqlen_q, nheads, headdim)``. For a contiguous cache,\n``k_cache``/``v_cache`` are ``(batch, seqlen_cache, nheads_k, headdim)`` and\n``cache_seqlens`` (int or ``(batch,)``) is the number of valid cache entries\n*before* appending. If ``k``/``v`` (the new tokens) are given, they are\nwritten into the cache in place. The query attends to the valid prefix\n(``cache_seqlens`` + ``seqlen_q``) with bottom-right causal alignment.\n\nNotes / current limitations:\n- Rotary embedding is not applied here; pass ``q``/``k`` already rotated\n (``rotary_cos``/``rotary_sin`` are rejected).\n- Paged caches are addressed via ``block_table`` (forwarded to the split-KV\n varlen kernel); in-cache append for paged layouts must be done by the\n caller (pass ``k=v=None``).",
|
| 251 |
+
"signature": {
|
| 252 |
+
"parameters": [
|
| 253 |
+
{
|
| 254 |
+
"name": "q",
|
| 255 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 256 |
+
"default": null,
|
| 257 |
+
"annotation": "torch.Tensor"
|
| 258 |
+
},
|
| 259 |
+
{
|
| 260 |
+
"name": "k_cache",
|
| 261 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 262 |
+
"default": null,
|
| 263 |
+
"annotation": "torch.Tensor"
|
| 264 |
+
},
|
| 265 |
+
{
|
| 266 |
+
"name": "v_cache",
|
| 267 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 268 |
+
"default": null,
|
| 269 |
+
"annotation": "torch.Tensor"
|
| 270 |
+
},
|
| 271 |
+
{
|
| 272 |
+
"name": "k",
|
| 273 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 274 |
+
"default": "None",
|
| 275 |
+
"annotation": "torch.Tensor | None"
|
| 276 |
+
},
|
| 277 |
+
{
|
| 278 |
+
"name": "v",
|
| 279 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 280 |
+
"default": "None",
|
| 281 |
+
"annotation": "torch.Tensor | None"
|
| 282 |
+
},
|
| 283 |
+
{
|
| 284 |
+
"name": "rotary_cos",
|
| 285 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 286 |
+
"default": "None",
|
| 287 |
+
"annotation": "torch.Tensor | None"
|
| 288 |
+
},
|
| 289 |
+
{
|
| 290 |
+
"name": "rotary_sin",
|
| 291 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 292 |
+
"default": "None",
|
| 293 |
+
"annotation": "torch.Tensor | None"
|
| 294 |
+
},
|
| 295 |
+
{
|
| 296 |
+
"name": "cache_seqlens",
|
| 297 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 298 |
+
"default": "None",
|
| 299 |
+
"annotation": null
|
| 300 |
+
},
|
| 301 |
+
{
|
| 302 |
+
"name": "block_table",
|
| 303 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 304 |
+
"default": "None",
|
| 305 |
+
"annotation": "torch.Tensor | None"
|
| 306 |
+
},
|
| 307 |
+
{
|
| 308 |
+
"name": "softmax_scale",
|
| 309 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 310 |
+
"default": "None",
|
| 311 |
+
"annotation": "float | None"
|
| 312 |
+
},
|
| 313 |
+
{
|
| 314 |
+
"name": "causal",
|
| 315 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 316 |
+
"default": "True",
|
| 317 |
+
"annotation": "bool"
|
| 318 |
+
},
|
| 319 |
+
{
|
| 320 |
+
"name": "window_size",
|
| 321 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 322 |
+
"default": "(-1, -1, 0)",
|
| 323 |
+
"annotation": "Tuple[int, ...]"
|
| 324 |
+
},
|
| 325 |
+
{
|
| 326 |
+
"name": "s_aux",
|
| 327 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 328 |
+
"default": "None",
|
| 329 |
+
"annotation": "torch.Tensor | None"
|
| 330 |
+
},
|
| 331 |
+
{
|
| 332 |
+
"name": "return_lse",
|
| 333 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 334 |
+
"default": "False",
|
| 335 |
+
"annotation": "bool"
|
| 336 |
+
},
|
| 337 |
+
{
|
| 338 |
+
"name": "kwargs",
|
| 339 |
+
"kind": "VAR_KEYWORD",
|
| 340 |
+
"default": null,
|
| 341 |
+
"annotation": null
|
| 342 |
+
}
|
| 343 |
+
],
|
| 344 |
+
"return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]"
|
| 345 |
+
}
|
| 346 |
+
}
|
| 347 |
+
],
|
| 348 |
+
"layers": []
|
| 349 |
+
}
|
build/torch212-rocm72-x86_64-linux/__init__.py
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""AITER Composable-Kernel FlashAttention kernels for AMD ROCm (compiled HIP).
|
| 2 |
+
|
| 3 |
+
Compiled (Composable Kernel / HIP) counterpart to the Triton-based
|
| 4 |
+
``aiter-flash-attn`` package, repackaged from the CK FMHA forward path of the
|
| 5 |
+
`ROCm/aiter <https://github.com/ROCm/aiter>`_ project as a self-contained
|
| 6 |
+
Hugging Face Hub kernel. Exposes the FlashAttention forward entry points with
|
| 7 |
+
learnable attention sink support (e.g. gpt-oss).
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
from .mha import (
|
| 11 |
+
flash_attn_func,
|
| 12 |
+
flash_attn_varlen_func,
|
| 13 |
+
flash_attn_with_kvcache,
|
| 14 |
+
)
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
__kernel_metadata__ = {
|
| 18 |
+
"license": "mit",
|
| 19 |
+
}
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
__all__ = [
|
| 23 |
+
"__kernel_metadata__",
|
| 24 |
+
"flash_attn_func",
|
| 25 |
+
"flash_attn_varlen_func",
|
| 26 |
+
"flash_attn_with_kvcache",
|
| 27 |
+
]
|
build/torch212-rocm72-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:7b2047a691f2945147456fef163603452d23707ec638cf2ea76fde4f596571ba
|
| 3 |
+
size 189896064
|
build/torch212-rocm72-x86_64-linux/_ops.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
from . import _aiter_flash_attn_ck_rocm_fe2bb66
|
| 3 |
+
ops = torch.ops._aiter_flash_attn_ck_rocm_fe2bb66
|
| 4 |
+
|
| 5 |
+
def add_op_namespace_prefix(op_name: str):
|
| 6 |
+
"""
|
| 7 |
+
Prefix op by namespace.
|
| 8 |
+
"""
|
| 9 |
+
return f"_aiter_flash_attn_ck_rocm_fe2bb66::{op_name}"
|
build/torch212-rocm72-x86_64-linux/metadata.json
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "aiter-flash-attn-ck",
|
| 3 |
+
"id": "_aiter_flash_attn_ck_rocm_fe2bb66",
|
| 4 |
+
"version": 1,
|
| 5 |
+
"kernels-minver": "0.14.0",
|
| 6 |
+
"license": "MIT",
|
| 7 |
+
"python-depends": [],
|
| 8 |
+
"kernel-depends": [],
|
| 9 |
+
"backend": {
|
| 10 |
+
"type": "rocm",
|
| 11 |
+
"archs": [
|
| 12 |
+
"gfx942",
|
| 13 |
+
"gfx950"
|
| 14 |
+
]
|
| 15 |
+
},
|
| 16 |
+
"digest": {
|
| 17 |
+
"algorithm": "sha256",
|
| 18 |
+
"files": {
|
| 19 |
+
"__init__.py": "7wj1GTm2bqTLLCyT4xLuAk/+JWYrhINjY/ODMNWhgZ0=",
|
| 20 |
+
"_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so": "eyBHppHylFFHRW/vFjYDRS0jcH7GOM8up2/eT1llcbo=",
|
| 21 |
+
"_ops.py": "4WkdYDWWJp60zep+oW/5ym9vIeVMNfrgRCkP3o/w7yU=",
|
| 22 |
+
"mha.py": "zCAjY4O+DiCzmEL+W9sqXxZaqjsj4RJbWFcyPpUZejk=",
|
| 23 |
+
"symbols.json": "rhnF4H8Jsb6I+YPxIVFMgnLcTQJODEXbZ7ge77NZXAU="
|
| 24 |
+
}
|
| 25 |
+
},
|
| 26 |
+
"provenance": {
|
| 27 |
+
"kernel-builder": {
|
| 28 |
+
"version": "0.18.0-dev0",
|
| 29 |
+
"commit": "d054906a13710b811b96697b4635f03020842733",
|
| 30 |
+
"dirty": false
|
| 31 |
+
},
|
| 32 |
+
"kernel": {
|
| 33 |
+
"commit": "fe2bb66fee6c2a77eccf08a0508dc57428d488ed",
|
| 34 |
+
"dirty": false
|
| 35 |
+
}
|
| 36 |
+
}
|
| 37 |
+
}
|
build/torch212-rocm72-x86_64-linux/metadata.json.sigstore
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json", "verificationMaterial":{"certificate":{"rawBytes":"MIIHLTCCBrKgAwIBAgIUQ45I/pFC4gf04I3vc+3x0g7A174wCgYIKoZIzj0EAwMwNzEVMBMGA1UEChMMc2lnc3RvcmUuZGV2MR4wHAYDVQQDExVzaWdzdG9yZS1pbnRlcm1lZGlhdGUwHhcNMjYxMDE5MjIzMTE3WhcNMjYxMDE5MjI0MTE3WjAAMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEzjsfKDVfgfG/aQDPwqQuhwftoZEZK9S5hBiZEa1B80IEyXRgtQ/mvsOwoTYJ9mvxC1+nNSGJdimuEs+feeY6f6OCBdEwggXNMA4GA1UdDwEB/wQEAwIHgDATBgNVHSUEDDAKBggrBgEFBQcDAzAfBgNVHSMEGDAWgBTf0+nPViQRlvmo2OkoVaLGLhhkPzBrBgNVHREBAf8EYTBfhl1odHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkvLmdpdGh1Yi93b3JrZmxvd3MvYnVpbGQueWFtbEByZWZzL2hlYWRzL21haW4wOQYKKwYBBAGDvzABAQQraHR0cHM6Ly90b2tlbi5hY3Rpb25zLmdpdGh1YnVzZXJjb250ZW50LmNvbTAfBgorBgEEAYO/MAECBBF3b3JrZmxvd19kaXNwYXRjaDA2BgorBgEEAYO/MAEDBChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMBMGCisGAQQBg78wAQQEBUJ1aWxkMCsGCisGAQQBg78wAQUEHWh1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5MB0GCisGAQQBg78wAQYED3JlZnMvaGVhZHMvbWFpbjA7BgorBgEEAYO/MAEIBC0MK2h0dHBzOi8vdG9rZW4uYWN0aW9ucy5naXRodWJ1c2VyY29udGVudC5jb20wbQYKKwYBBAGDvzABCQRfDF1odHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkvLmdpdGh1Yi93b3JrZmxvd3MvYnVpbGQueWFtbEByZWZzL2hlYWRzL21haW4wOAYKKwYBBAGDvzABCgQqDChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMBsGCisGAQQBg78wAQsEDQwLc2VsZi1ob3N0ZWQwQAYKKwYBBAGDvzABDAQyDDBodHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkwOAYKKwYBBAGDvzABDQQqDChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMB8GCisGAQQBg78wAQ4EEQwPcmVmcy9oZWFkcy9tYWluMBoGCisGAQQBg78wAQ8EDAwKMTA3MTQ3NTUyOTAuBgorBgEEAYO/MAEQBCAMHmh0dHBzOi8vZ2l0aHViLmNvbS9odWdnaW5nZmFjZTAYBgorBgEEAYO/MAERBAoMCDI1NzIwNzQzMG0GCisGAQQBg78wARIEXwxdaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5Ly5naXRodWIvd29ya2Zsb3dzL2J1aWxkLnlhbWxAcmVmcy9oZWFkcy9tYWluMDgGCisGAQQBg78wARMEKgwoZmUyYmI2NmZlZTZjMmE3N2VjY2YwOGEwNTA4ZGM1NzQyOGQ0ODhlZDAhBgorBgEEAYO/MAEUBBMMEXdvcmtmbG93X2Rpc3BhdGNoMGQGCisGAQQBg78wARUEVgxUaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5L2FjdGlvbnMvcnVucy8zOTIzOTUxODEwNC9hdHRlbXB0cy8xMBYGCisGAQQBg78wARYECAwGcHVibGljMEYGCisGAQQBg78wARgEOAw2cmVwbzpodWdnaW5nZmFjZS9rZXJuZWxzLWNvbW11bml0eTpyZWY6cmVmcy9oZWFkcy9tYWluMIGKBgorBgEEAdZ5AgQCBHwEegB4AHYA3T0wasbHETJjGR4cmWc3AqJKXrjePK3/h4pygC8p7o4AAAGhVkphtgAABAMARzBFAiEA/V/XQ1zoAiy9Jx6LwsG2v9te8QMMdKgo5lxz6/CHUYYCIEBZ2id17T5aK+gexeVyI/uX1eFwmxl5wWBqdSKkHIfWMAoGCCqGSM49BAMDA2kAMGYCMQCtCXUfftdOhO4fxL+pXAyRkt9yANatw2Of1I2oXJa2N5+Ig+NVi5LLNvVkIwur1dMCMQDbzUgi38ARNQqw4NGCku4uStNr02qaF3jeDjSWPjcHfJfFxwDebjKCISt/W2u42Qo="}, "tlogEntries":[{"logIndex":"3327212104", "logId":{"keyId":"wNI9atQGlz+VWfO6LRygH4QUfY/8W4RFwiT5i5WRgB0="}, "kindVersion":{"kind":"hashedrekord", "version":"0.0.1"}, "integratedTime":"1792449077", "inclusionPromise":{"signedEntryTimestamp":"MEYCIQDRlFftrJStusZSVz3NRwN5GmWooDwyhs6XhSTx9F5qyAIhAImpjdodpPfG6f+FY+9Zu/ShY0j1Tdx6nmld19W2EEbc"}, "inclusionProof":{"logIndex":"3205307842", "rootHash":"Zzz8n5Mp9ft5puTfBQIBlCPo6wcO7myrHF6c/DCLCto=", "treeSize":"3205307848", "hashes":["PNwoDRxfezGRvsQGmM0Kf8ZjXCh17m6718g830ihWTY=", "4W/Uty2EfVeMffxMat+M956so0Ydun9CqlURs6q1cTk=", "ZecfiCPMv2u2lSRP0ST3fdJpMAgVpdtdk/HMrR51Tzg=", "RCHihIv1d3ARj25cx7a6BlBmIRIvbqbcYga+rFtfimU=", "NFzeBDxwmGphfWEGD4k9bLG9D0Y0y1ZYfIWGhkLVpVs=", "F4ofUVGe4nSzmh+WwnM4OaVmZh/WQERaH+u/wt8i8FM=", "0aSE/O6jCq82vSwoybvpblRnnP72+1gEInKy6Wyc3E0=", "5egUlZ4Pi0Ix/qLyCDXocsWywJCHqrde/S06vOZvlOM=", "QsXufUpoKtSjKAB19K1G00umQpWvPHlBNuRGFiiVjQQ=", "QIMkwQbwAewhLJd//5FFwI3SnyNQBGjCgJrelfG8vhc=", "Et17CACUv9QN64irUJuBjsrgRGkGnnJ9xacRV/aBdwQ=", "lqgEwkxhDMs/gdIAE4lmjfMwg7nDmv7lda6O0LGliCw=", "HovTPBWzoErvUg1u5sIi3x1ZonZ7yJekVsBrmpomCIA=", "G+nk7mgE5h6dUmv2UY7NDJrY3WvrZY31raaO1AETt+o=", "qlTWNDzERnTBQVcHHXXVPjiUP0SsikGxfcxVlgqVFDo=", "gg9GNApNke3+zNhl4tsWtmIycCFrOMCB3lys6XuSQNQ=", "NlOclldmKPA1tMKIskqHAwuQDYHp31YwpK+OOMrMccc=", "qxzHanAzz57SDdmJe0B7bJK72NTIBbwEMGBBKvDOROw=", "xH/DCseLHr9eKoYT8qsORZK7zVdEGYWHuVtsVrD95wY="], "checkpoint":{"envelope":"rekor.sigstore.dev - 1193050959916656506\n3205307848\nZzz8n5Mp9ft5puTfBQIBlCPo6wcO7myrHF6c/DCLCto=\n\n— rekor.sigstore.dev wNI9ajBEAiAp++VhzyYu83HrKvp+GRExuyer5bXN7+E3gZBsz5SdWwIgf1ZNOW/vRsfmirbx4AdVrRpcwLjtog7l6WYnLs3jdHI=\n"}}, "canonicalizedBody":"eyJhcGlWZXJzaW9uIjoiMC4wLjEiLCJraW5kIjoiaGFzaGVkcmVrb3JkIiwic3BlYyI6eyJkYXRhIjp7Imhhc2giOnsiYWxnb3JpdGhtIjoic2hhMjU2IiwidmFsdWUiOiIzNTcxOTkxYjg4ZDIyMGFkMmUwMjU2OWUyNmIxYjQzY2U5MjAxNjU0M2M1OGE3MWFjOWJiYjEzOTQxZDEwYzQzIn19LCJzaWduYXR1cmUiOnsiY29udGVudCI6Ik1FVUNJRE1QTHZ0UEp2QVZvQ0lrL3dkY1hWSW1NYUNCbW1aNHZ3c2FtN1hiZ0hhcUFpRUFwbTFwOTNZeWdSVHBOdExHeUhYZnFtSnE1MmdnbElYcURKeHY0TEhxMzdVPSIsInB1YmxpY0tleSI6eyJjb250ZW50IjoiTFMwdExTMUNSVWRKVGlCRFJWSlVTVVpKUTBGVVJTMHRMUzB0Q2sxSlNVaE1WRU5EUW5KTFowRjNTVUpCWjBsVlVUUTFTUzl3UmtNMFoyWXdORWt6ZG1Nck0zZ3daemRCTVRjMGQwTm5XVWxMYjFwSmVtb3dSVUYzVFhjS1RucEZWazFDVFVkQk1WVkZRMmhOVFdNeWJHNWpNMUoyWTIxVmRWcEhWakpOVWpSM1NFRlpSRlpSVVVSRmVGWjZZVmRrZW1SSE9YbGFVekZ3WW01U2JBcGpiVEZzV2tkc2FHUkhWWGRJYUdOT1RXcFplRTFFUlRWTmFrbDZUVlJGTTFkb1kwNU5hbGw0VFVSRk5VMXFTVEJOVkVVelYycEJRVTFHYTNkRmQxbElDa3R2V2tsNmFqQkRRVkZaU1V0dldrbDZhakJFUVZGalJGRm5RVVY2YW5ObVMwUldabWRtUnk5aFVVUlFkM0ZSZFdoM1puUnZXa1ZhU3psVE5XaENhVm9LUldFeFFqZ3dTVVY1V0ZKbmRGRXZiWFp6VDNkdlZGbEtPVzEyZUVNeEsyNU9VMGRLWkdsdGRVVnpLMlpsWlZrMlpqWlBRMEprUlhkbloxaE9UVUUwUndwQk1WVmtSSGRGUWk5M1VVVkJkMGxJWjBSQlZFSm5UbFpJVTFWRlJFUkJTMEpuWjNKQ1owVkdRbEZqUkVGNlFXWkNaMDVXU0ZOTlJVZEVRVmRuUWxSbUNqQXJibEJXYVZGU2JIWnRiekpQYTI5V1lVeEhUR2hvYTFCNlFuSkNaMDVXU0ZKRlFrRm1PRVZaVkVKbWFHd3hiMlJJVW5kamVtOTJUREprY0dSSGFERUtXV2sxYW1JeU1IWmhTRlp1V2pKc2RWb3lXbWhaTWxWMllUSldlV0p0Vm5OamVURnFZakl4ZEdSWE5YQmtTR3QyVEcxa2NHUkhhREZaYVRrellqTktjZ3BhYlhoMlpETk5kbGx1Vm5CaVIxRjFaVmRHZEdKRlFubGFWMXA2VERKb2JGbFhVbnBNTWpGb1lWYzBkMDlSV1V0TGQxbENRa0ZIUkhaNlFVSkJVVkZ5Q21GSVVqQmpTRTAyVEhrNU1HSXlkR3hpYVRWb1dUTlNjR0l5TlhwTWJXUndaRWRvTVZsdVZucGFXRXBxWWpJMU1GcFhOVEJNYlU1MllsUkJaa0puYjNJS1FtZEZSVUZaVHk5TlFVVkRRa0pHTTJJelNuSmFiWGgyWkRFNWEyRllUbmRaV0ZKcVlVUkJNa0puYjNKQ1owVkZRVmxQTDAxQlJVUkNRMmh0V2xSS2FRcFphbGt5V20xV2JFNXRUWGxaVkdNeldsZE9hbHBxUVRSWlZFRXhUVVJvYTFsNlZUTk9SRWswV2tSUk5FOUhWbXROUWsxSFEybHpSMEZSVVVKbk56aDNDa0ZSVVVWQ1ZVb3hZVmQ0YTAxRGMwZERhWE5IUVZGUlFtYzNPSGRCVVZWRlNGZG9NVm95WkhCaWJXUnRXVmRPYkV3eWRHeGpiVFZzWWtoTmRGa3lPWFFLWWxoV2RXRllValZOUWpCSFEybHpSMEZSVVVKbk56aDNRVkZaUlVRelNteGFiazEyWVVkV2FGcElUWFppVjBad1ltcEJOMEpuYjNKQ1owVkZRVmxQTHdwTlFVVkpRa013VFVzeWFEQmtTRUo2VDJrNGRtUkhPWEphVnpSMVdWZE9NR0ZYT1hWamVUVnVZVmhTYjJSWFNqRmpNbFo1V1RJNWRXUkhWblZrUXpWcUNtSXlNSGRpVVZsTFMzZFpRa0pCUjBSMmVrRkNRMUZTWmtSR01XOWtTRkozWTNwdmRrd3laSEJrUjJneFdXazFhbUl5TUhaaFNGWnVXakpzZFZveVdtZ0tXVEpWZG1FeVZubGliVlp6WTNreGFtSXlNWFJrVnpWd1pFaHJka3h0WkhCa1IyZ3hXV2s1TTJJelNuSmFiWGgyWkROTmRsbHVWbkJpUjFGMVpWZEdkQXBpUlVKNVdsZGFla3d5YUd4WlYxSjZUREl4YUdGWE5IZFBRVmxMUzNkWlFrSkJSMFIyZWtGQ1EyZFJjVVJEYUcxYVZFcHBXV3BaTWxwdFZteE9iVTE1Q2xsVVl6TmFWMDVxV21wQk5GbFVRVEZOUkdocldYcFZNMDVFU1RSYVJGRTBUMGRXYTAxQ2MwZERhWE5IUVZGUlFtYzNPSGRCVVhORlJGRjNUR015Vm5NS1dta3hiMkl6VGpCYVYxRjNVVUZaUzB0M1dVSkNRVWRFZG5wQlFrUkJVWGxFUkVKdlpFaFNkMk42YjNaTU1tUndaRWRvTVZscE5XcGlNakIyWVVoV2JncGFNbXgxV2pKYWFGa3lWWFpoTWxaNVltMVdjMk41TVdwaU1qRjBaRmMxY0dSSWEzZFBRVmxMUzNkWlFrSkJSMFIyZWtGQ1JGRlJjVVJEYUcxYVZFcHBDbGxxV1RKYWJWWnNUbTFOZVZsVVl6TmFWMDVxV21wQk5GbFVRVEZOUkdocldYcFZNMDVFU1RSYVJGRTBUMGRXYTAxQ09FZERhWE5IUVZGUlFtYzNPSGNLUVZFMFJVVlJkMUJqYlZadFkzazViMXBYUm10amVUbDBXVmRzZFUxQ2IwZERhWE5IUVZGUlFtYzNPSGRCVVRoRlJFRjNTMDFVUVROTlZGRXpUbFJWZVFwUFZFRjFRbWR2Y2tKblJVVkJXVTh2VFVGRlVVSkRRVTFJYldnd1pFaENlazlwT0haYU1td3dZVWhXYVV4dFRuWmlVemx2WkZka2JtRlhOVzVhYlVacUNscFVRVmxDWjI5eVFtZEZSVUZaVHk5TlFVVlNRa0Z2VFVORVNURk9la2wzVG5wUmVrMUhNRWREYVhOSFFWRlJRbWMzT0hkQlVrbEZXSGQ0WkdGSVVqQUtZMGhOTmt4NU9XNWhXRkp2WkZkSmRWa3lPWFJNTW1neFdqSmtjR0p0WkcxWlYwNXNUREowYkdOdE5XeGlTRTEwV1RJNWRHSllWblZoV0ZJMVRIazFiZ3BoV0ZKdlpGZEpkbVF5T1hsaE1scHpZak5rZWt3eVNqRmhWM2hyVEc1c2FHSlhlRUZqYlZadFkzazViMXBYUm10amVUbDBXVmRzZFUxRVowZERhWE5IQ2tGUlVVSm5OemgzUVZKTlJVdG5kMjlhYlZWNVdXMUpNazV0V214YVZGcHFUVzFGTTA0eVZtcFpNbGwzVDBkRmQwNVVRVFJhUjAweFRucFJlVTlIVVRBS1QwUm9iRnBFUVdoQ1oyOXlRbWRGUlVGWlR5OU5RVVZWUWtKTlRVVllaSFpqYlhSdFlrYzVNMWd5VW5Cak0wSm9aRWRPYjAxSFVVZERhWE5IUVZGUlFncG5OemgzUVZKVlJWWm5lRlZoU0ZJd1kwaE5Oa3g1T1c1aFdGSnZaRmRKZFZreU9YUk1NbWd4V2pKa2NHSnRaRzFaVjA1c1RESjBiR050Tld4aVNFMTBDbGt5T1hSaVdGWjFZVmhTTlV3eVJtcGtSMngyWW01TmRtTnVWblZqZVRoNlQxUkplazlVVlhoUFJFVjNUa001YUdSSVVteGlXRUl3WTNrNGVFMUNXVWNLUTJselIwRlJVVUpuTnpoM1FWSlpSVU5CZDBkalNGWnBZa2RzYWsxRldVZERhWE5IUVZGUlFtYzNPSGRCVW1kRlQwRjNNbU50Vm5kaWVuQnZaRmRrYmdwaFZ6VnVXbTFHYWxwVE9YSmFXRXAxV2xkNGVreFhUblppVnpFeFltMXNNR1ZVY0hsYVYxazJZMjFXYldONU9XOWFWMFpyWTNrNWRGbFhiSFZOU1VkTENrSm5iM0pDWjBWRlFXUmFOVUZuVVVOQ1NIZEZaV2RDTkVGSVdVRXpWREIzWVhOaVNFVlVTbXBIVWpSamJWZGpNMEZ4U2t0WWNtcGxVRXN6TDJnMGNIa0taME00Y0Rkdk5FRkJRVWRvVm10d2FIUm5RVUZDUVUxQlVucENSa0ZwUlVFdlZpOVlVVEY2YjBGcGVUbEtlRFpNZDNOSE1uWTVkR1U0VVUxTlpFdG5id28xYkhoNk5pOURTRlZaV1VOSlJVSmFNbWxrTVRkVU5XRkxLMmRsZUdWV2VVa3ZkVmd4WlVaM2JYaHNOWGRYUW5Ga1UwdHJTRWxtVjAxQmIwZERRM0ZIQ2xOTk5EbENRVTFFUVRKclFVMUhXVU5OVVVOMFExaFZabVowWkU5b1R6Um1lRXdyY0ZoQmVWSnJkRGw1UVU1aGRIY3lUMll4U1RKdldFcGhNazQxSzBrS1p5dE9WbWsxVEV4T2RsWnJTWGQxY2pGa1RVTk5VVVJpZWxWbmFUTTRRVkpPVVhGM05FNUhRMnQxTkhWVGRFNXlNREp4WVVZemFtVkVhbE5YVUdwalNBcG1TbVpHZUhkRVpXSnFTME5KVTNRdlZ6SjFOREpSYnowS0xTMHRMUzFGVGtRZ1EwVlNWRWxHU1VOQlZFVXRMUzB0TFFvPSJ9fX19"}], "timestampVerificationData":{"rfc3161Timestamps":[{"signedTimestamp":"MIICyTADAgEAMIICwAYJKoZIhvcNAQcCoIICsTCCAq0CAQMxDTALBglghkgBZQMEAgEwgbgGCyqGSIb3DQEJEAEEoIGoBIGlMIGiAgEBBgkrBgEEAYO/MAIwMTANBglghkgBZQMEAgEFAAQgmTkV835pgwxkvMNukL50F/XFs1cK4XlYhGVjIf6CtBsCFQCFgrdHkuvKvwpQj8RpZZmBpRIUlhgPMjAyNjEwMTkyMjMxMTdaMAMCAQGgMqQwMC4xFTATBgNVBAoTDHNpZ3N0b3JlLmRldjEVMBMGA1UEAxMMc2lnc3RvcmUtdHNhoAAxggHaMIIB1gIBATBRMDkxFTATBgNVBAoTDHNpZ3N0b3JlLmRldjEgMB4GA1UEAxMXc2lnc3RvcmUtdHNhLXNlbGZzaWduZWQCFDoTVC8MkGHuvMFDL8uKjosqI4sMMAsGCWCGSAFlAwQCAaCB/DAaBgkqhkiG9w0BCQMxDQYLKoZIhvcNAQkQAQQwHAYJKoZIhvcNAQkFMQ8XDTI2MTAxOTIyMzExN1owLwYJKoZIhvcNAQkEMSIEIJRd9HQUlnO59ivgTId0aXPDBwASdkAbCnMh269+NWAGMIGOBgsqhkiG9w0BCRACLzF/MH0wezB5BCCF+Se8B6tiysO0Q1bBDvyBssaIP9p6uebYcNnROs0FtzBVMD2kOzA5MRUwEwYDVQQKEwxzaWdzdG9yZS5kZXYxIDAeBgNVBAMTF3NpZ3N0b3JlLXRzYS1zZWxmc2lnbmVkAhQ6E1QvDJBh7rzBQy/Lio6LKiOLDDAKBggqhkjOPQQDAgRmMGQCMB8mlkizvZTubl6yOTamotJcCj+VOyU8JO51/iJzccSbUZEK7a58vYbacjQtKz0LdQIwPO96KwU6xzAQ+JfLYq0wy1m87aCJg68U22OyZ0sqwJvVatEW1hTkicd7Gh5GBX98"}]}}, "messageSignature":{"messageDigest":{"algorithm":"SHA2_256", "digest":"NXGZG4jSIK0uAlaeJrG0POkgFlQ8WKcaybuxOUHRDEM="}, "signature":"MEUCIDMPLvtPJvAVoCIk/wdcXVImMaCBmmZ4vwsam7XbgHaqAiEApm1p93YygRTpNtLGyHXfqmJq52gglIXqDJxv4LHq37U="}}
|
build/torch212-rocm72-x86_64-linux/mha.py
ADDED
|
@@ -0,0 +1,332 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""FlashAttention forward entry points backed by aiter's Composable-Kernel FMHA.
|
| 2 |
+
|
| 3 |
+
This is the compiled (HIP / Composable Kernel) counterpart to the Triton
|
| 4 |
+
``aiter-flash-attn`` package. Only the *forward* pass is provided (dense and
|
| 5 |
+
variable-length), including support for learnable attention sinks via
|
| 6 |
+
``window_size[2]`` / ``s_aux`` (e.g. gpt-oss). The op set ships exclusively
|
| 7 |
+
the CK ``mha_fwd`` / ``mha_varlen_fwd`` kernels, so the sink path is always
|
| 8 |
+
taken (no fall-through to a non-sink ASM kernel).
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
from typing import Optional, Tuple, Union
|
| 12 |
+
|
| 13 |
+
import torch
|
| 14 |
+
from torch import Tensor
|
| 15 |
+
|
| 16 |
+
from ._ops import ops
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def _round_multiple(x: int, m: int) -> int:
|
| 20 |
+
return (x + m - 1) // m * m
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def _maybe_pad_headdim(t: Tensor, target: int) -> Tensor:
|
| 24 |
+
og = t.size(-1)
|
| 25 |
+
if og == target:
|
| 26 |
+
return t
|
| 27 |
+
return torch.nn.functional.pad(t, [0, target - og])
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def _unpack_window(window_size) -> Tuple[int, int, int]:
|
| 31 |
+
left = int(window_size[0])
|
| 32 |
+
right = int(window_size[1])
|
| 33 |
+
sink = int(window_size[2]) if len(window_size) == 3 else 0
|
| 34 |
+
return left, right, sink
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def _prep_sink(s_aux: Optional[Tensor], nheads: int, device) -> Optional[Tensor]:
|
| 38 |
+
if s_aux is None:
|
| 39 |
+
return None
|
| 40 |
+
assert s_aux.device == device, "s_aux must be on the same device as q"
|
| 41 |
+
assert s_aux.shape[0] == nheads, "s_aux must have shape [nheads_q]"
|
| 42 |
+
if s_aux.dtype != torch.float32:
|
| 43 |
+
s_aux = s_aux.to(torch.float32)
|
| 44 |
+
return s_aux
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def _effective_sink_size(sink_size: int, s_aux: Optional[Tensor]) -> int:
|
| 48 |
+
# The CK kernel only takes the sink path when sink_size > 0. Callers using
|
| 49 |
+
# the transformers attention interface pass the per-head sink logits as
|
| 50 |
+
# ``s_aux`` but leave ``window_size`` a 2-tuple (sink_size == 0), so enable
|
| 51 |
+
# a single learnable sink automatically when s_aux is given.
|
| 52 |
+
if s_aux is not None and sink_size == 0:
|
| 53 |
+
return 1
|
| 54 |
+
return sink_size
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def flash_attn_func(
|
| 58 |
+
q: Tensor,
|
| 59 |
+
k: Tensor,
|
| 60 |
+
v: Tensor,
|
| 61 |
+
dropout_p: float = 0.0,
|
| 62 |
+
softmax_scale: Optional[float] = None,
|
| 63 |
+
causal: bool = False,
|
| 64 |
+
window_size: Tuple[int, ...] = (-1, -1, 0),
|
| 65 |
+
bias: Optional[Tensor] = None,
|
| 66 |
+
alibi_slopes: Optional[Tensor] = None,
|
| 67 |
+
return_lse: bool = False,
|
| 68 |
+
return_attn_probs: bool = False,
|
| 69 |
+
s_aux: Optional[Tensor] = None,
|
| 70 |
+
**kwargs,
|
| 71 |
+
) -> Union[Tensor, Tuple[Tensor, ...]]:
|
| 72 |
+
"""Dense FlashAttention forward.
|
| 73 |
+
|
| 74 |
+
``q, k, v`` have shape ``(batch, seqlen, nheads, headdim)``. ``window_size``
|
| 75 |
+
is ``(left, right, sink_size)``; ``s_aux`` is an optional ``(nheads_q,)``
|
| 76 |
+
fp32 tensor of learnable sink logits. Extra keyword arguments (e.g. those
|
| 77 |
+
passed by the transformers flash-attention interface) are ignored.
|
| 78 |
+
"""
|
| 79 |
+
if softmax_scale is None:
|
| 80 |
+
softmax_scale = q.shape[-1] ** (-0.5)
|
| 81 |
+
window_size_left, window_size_right, sink_size = _unpack_window(window_size)
|
| 82 |
+
|
| 83 |
+
nheads_q = q.size(2)
|
| 84 |
+
s_aux = _prep_sink(s_aux, nheads_q, q.device)
|
| 85 |
+
sink_size = _effective_sink_size(sink_size, s_aux)
|
| 86 |
+
|
| 87 |
+
head_q_og = q.size(-1)
|
| 88 |
+
head_v_og = v.size(-1)
|
| 89 |
+
head_q = _round_multiple(head_q_og, 8)
|
| 90 |
+
head_v = _round_multiple(head_v_og, 8)
|
| 91 |
+
q = _maybe_pad_headdim(q, head_q)
|
| 92 |
+
k = _maybe_pad_headdim(k, head_q)
|
| 93 |
+
v = _maybe_pad_headdim(v, head_v)
|
| 94 |
+
|
| 95 |
+
return_softmax = return_attn_probs and dropout_p > 0
|
| 96 |
+
out, softmax_lse, S_dmask, _ = ops.mha_fwd(
|
| 97 |
+
q,
|
| 98 |
+
k,
|
| 99 |
+
v,
|
| 100 |
+
dropout_p,
|
| 101 |
+
softmax_scale,
|
| 102 |
+
causal,
|
| 103 |
+
window_size_left,
|
| 104 |
+
window_size_right,
|
| 105 |
+
sink_size,
|
| 106 |
+
return_lse or return_attn_probs,
|
| 107 |
+
return_softmax,
|
| 108 |
+
None, # cu_seqlens_q
|
| 109 |
+
None, # cu_seqlens_kv
|
| 110 |
+
None, # out
|
| 111 |
+
bias,
|
| 112 |
+
alibi_slopes,
|
| 113 |
+
None, # q_descale
|
| 114 |
+
None, # k_descale
|
| 115 |
+
None, # v_descale
|
| 116 |
+
s_aux,
|
| 117 |
+
None, # gen
|
| 118 |
+
)
|
| 119 |
+
|
| 120 |
+
out = out[..., :head_v_og]
|
| 121 |
+
if not (return_lse or return_attn_probs):
|
| 122 |
+
return out
|
| 123 |
+
result = (out,)
|
| 124 |
+
if return_lse:
|
| 125 |
+
result = result + (softmax_lse,)
|
| 126 |
+
if return_attn_probs:
|
| 127 |
+
result = result + (S_dmask,)
|
| 128 |
+
return result
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
def flash_attn_varlen_func(
|
| 132 |
+
q: Tensor,
|
| 133 |
+
k: Tensor,
|
| 134 |
+
v: Tensor,
|
| 135 |
+
cu_seqlens_q: Tensor,
|
| 136 |
+
cu_seqlens_k: Tensor,
|
| 137 |
+
max_seqlen_q: int,
|
| 138 |
+
max_seqlen_k: int,
|
| 139 |
+
dropout_p: float = 0.0,
|
| 140 |
+
softmax_scale: Optional[float] = None,
|
| 141 |
+
causal: bool = False,
|
| 142 |
+
window_size: Tuple[int, ...] = (-1, -1, 0),
|
| 143 |
+
bias: Optional[Tensor] = None,
|
| 144 |
+
alibi_slopes: Optional[Tensor] = None,
|
| 145 |
+
return_lse: bool = False,
|
| 146 |
+
return_attn_probs: bool = False,
|
| 147 |
+
block_table: Optional[Tensor] = None,
|
| 148 |
+
min_seqlen_q: int = 0,
|
| 149 |
+
logits_soft_cap: float = 0.0,
|
| 150 |
+
zero_tensors: bool = False,
|
| 151 |
+
cu_seqlens_q_padded: Optional[Tensor] = None,
|
| 152 |
+
cu_seqlens_k_padded: Optional[Tensor] = None,
|
| 153 |
+
s_aux: Optional[Tensor] = None,
|
| 154 |
+
**kwargs,
|
| 155 |
+
) -> Union[Tensor, Tuple[Tensor, ...]]:
|
| 156 |
+
"""Variable-length / packed FlashAttention forward.
|
| 157 |
+
|
| 158 |
+
``q`` has shape ``(total_q, nheads, headdim)``; ``cu_seqlens_*`` are int32
|
| 159 |
+
cumulative offsets of shape ``(batch + 1,)``. Sinks are supported through
|
| 160 |
+
``window_size[2]`` / ``s_aux`` as in :func:`flash_attn_func`. Extra keyword
|
| 161 |
+
arguments (e.g. from the transformers flash-attention interface) are ignored.
|
| 162 |
+
"""
|
| 163 |
+
if softmax_scale is None:
|
| 164 |
+
softmax_scale = q.shape[-1] ** (-0.5)
|
| 165 |
+
window_size_left, window_size_right, sink_size = _unpack_window(window_size)
|
| 166 |
+
|
| 167 |
+
nheads_q = q.size(1)
|
| 168 |
+
s_aux = _prep_sink(s_aux, nheads_q, q.device)
|
| 169 |
+
sink_size = _effective_sink_size(sink_size, s_aux)
|
| 170 |
+
|
| 171 |
+
head_q_og = q.size(-1)
|
| 172 |
+
head_v_og = v.size(-1)
|
| 173 |
+
head_q = _round_multiple(head_q_og, 8)
|
| 174 |
+
head_v = _round_multiple(head_v_og, 8)
|
| 175 |
+
q = _maybe_pad_headdim(q, head_q)
|
| 176 |
+
k = _maybe_pad_headdim(k, head_q)
|
| 177 |
+
v = _maybe_pad_headdim(v, head_v)
|
| 178 |
+
|
| 179 |
+
return_softmax = return_attn_probs and dropout_p > 0
|
| 180 |
+
out, softmax_lse, S_dmask, _ = ops.mha_varlen_fwd(
|
| 181 |
+
q,
|
| 182 |
+
k,
|
| 183 |
+
v,
|
| 184 |
+
cu_seqlens_q,
|
| 185 |
+
cu_seqlens_k,
|
| 186 |
+
max_seqlen_q,
|
| 187 |
+
max_seqlen_k,
|
| 188 |
+
min_seqlen_q,
|
| 189 |
+
dropout_p,
|
| 190 |
+
softmax_scale,
|
| 191 |
+
logits_soft_cap,
|
| 192 |
+
zero_tensors,
|
| 193 |
+
causal,
|
| 194 |
+
window_size_left,
|
| 195 |
+
window_size_right,
|
| 196 |
+
sink_size,
|
| 197 |
+
return_lse or return_attn_probs,
|
| 198 |
+
return_softmax,
|
| 199 |
+
None, # out
|
| 200 |
+
block_table,
|
| 201 |
+
bias,
|
| 202 |
+
alibi_slopes,
|
| 203 |
+
None, # q_descale
|
| 204 |
+
None, # k_descale
|
| 205 |
+
None, # v_descale
|
| 206 |
+
None, # gen
|
| 207 |
+
cu_seqlens_q_padded,
|
| 208 |
+
cu_seqlens_k_padded,
|
| 209 |
+
s_aux,
|
| 210 |
+
)
|
| 211 |
+
|
| 212 |
+
out = out[..., :head_v_og]
|
| 213 |
+
if not (return_lse or return_attn_probs):
|
| 214 |
+
return out
|
| 215 |
+
result = (out,)
|
| 216 |
+
if return_lse:
|
| 217 |
+
result = result + (softmax_lse,)
|
| 218 |
+
if return_attn_probs:
|
| 219 |
+
result = result + (S_dmask,)
|
| 220 |
+
return result
|
| 221 |
+
|
| 222 |
+
|
| 223 |
+
def _append_to_contiguous_cache(k_cache, v_cache, k, v, cache_seqlens, Sq):
|
| 224 |
+
"""In-place append of new k/v into a contiguous cache at per-batch offsets."""
|
| 225 |
+
B = k_cache.shape[0]
|
| 226 |
+
if isinstance(cache_seqlens, int):
|
| 227 |
+
cache_seqlens = torch.full((B,), cache_seqlens, device=k_cache.device, dtype=torch.long)
|
| 228 |
+
pos = cache_seqlens.view(B, 1).to(torch.long) + torch.arange(Sq, device=k_cache.device).view(1, Sq)
|
| 229 |
+
bidx = torch.arange(B, device=k_cache.device).view(B, 1).expand(B, Sq)
|
| 230 |
+
k_cache[bidx, pos] = k.to(k_cache.dtype)
|
| 231 |
+
v_cache[bidx, pos] = v.to(v_cache.dtype)
|
| 232 |
+
|
| 233 |
+
|
| 234 |
+
def flash_attn_with_kvcache(
|
| 235 |
+
q: Tensor,
|
| 236 |
+
k_cache: Tensor,
|
| 237 |
+
v_cache: Tensor,
|
| 238 |
+
k: Optional[Tensor] = None,
|
| 239 |
+
v: Optional[Tensor] = None,
|
| 240 |
+
rotary_cos: Optional[Tensor] = None,
|
| 241 |
+
rotary_sin: Optional[Tensor] = None,
|
| 242 |
+
cache_seqlens=None,
|
| 243 |
+
block_table: Optional[Tensor] = None,
|
| 244 |
+
softmax_scale: Optional[float] = None,
|
| 245 |
+
causal: bool = True,
|
| 246 |
+
window_size: Tuple[int, ...] = (-1, -1, 0),
|
| 247 |
+
s_aux: Optional[Tensor] = None,
|
| 248 |
+
return_lse: bool = False,
|
| 249 |
+
**kwargs,
|
| 250 |
+
) -> Union[Tensor, Tuple[Tensor, ...]]:
|
| 251 |
+
"""FlashAttention against a KV cache, built on the CK split-KV kernel.
|
| 252 |
+
|
| 253 |
+
``q`` is ``(batch, seqlen_q, nheads, headdim)``. For a contiguous cache,
|
| 254 |
+
``k_cache``/``v_cache`` are ``(batch, seqlen_cache, nheads_k, headdim)`` and
|
| 255 |
+
``cache_seqlens`` (int or ``(batch,)``) is the number of valid cache entries
|
| 256 |
+
*before* appending. If ``k``/``v`` (the new tokens) are given, they are
|
| 257 |
+
written into the cache in place. The query attends to the valid prefix
|
| 258 |
+
(``cache_seqlens`` + ``seqlen_q``) with bottom-right causal alignment.
|
| 259 |
+
|
| 260 |
+
Notes / current limitations:
|
| 261 |
+
- Rotary embedding is not applied here; pass ``q``/``k`` already rotated
|
| 262 |
+
(``rotary_cos``/``rotary_sin`` are rejected).
|
| 263 |
+
- Paged caches are addressed via ``block_table`` (forwarded to the split-KV
|
| 264 |
+
varlen kernel); in-cache append for paged layouts must be done by the
|
| 265 |
+
caller (pass ``k=v=None``).
|
| 266 |
+
"""
|
| 267 |
+
if rotary_cos is not None or rotary_sin is not None:
|
| 268 |
+
raise NotImplementedError(
|
| 269 |
+
"flash_attn_with_kvcache does not apply rotary; rotate q/k beforehand."
|
| 270 |
+
)
|
| 271 |
+
if softmax_scale is None:
|
| 272 |
+
softmax_scale = q.shape[-1] ** (-0.5)
|
| 273 |
+
window_size_left, window_size_right, sink_size = _unpack_window(window_size)
|
| 274 |
+
B, Sq, Hq, D = q.shape
|
| 275 |
+
s_aux = _prep_sink(s_aux, Hq, q.device)
|
| 276 |
+
sink_size = _effective_sink_size(sink_size, s_aux)
|
| 277 |
+
|
| 278 |
+
paged = block_table is not None
|
| 279 |
+
|
| 280 |
+
# ---- append new tokens into the cache (contiguous layout only) ----
|
| 281 |
+
if k is not None and not paged:
|
| 282 |
+
assert cache_seqlens is not None, "cache_seqlens required to append into the cache"
|
| 283 |
+
_append_to_contiguous_cache(k_cache, v_cache, k, v, cache_seqlens, Sq)
|
| 284 |
+
elif k is not None and paged:
|
| 285 |
+
raise NotImplementedError(
|
| 286 |
+
"in-cache append for paged layouts is not supported; append externally "
|
| 287 |
+
"and call with k=v=None."
|
| 288 |
+
)
|
| 289 |
+
|
| 290 |
+
appended = 0 if k is None else Sq
|
| 291 |
+
|
| 292 |
+
# ---- key lengths ----
|
| 293 |
+
if cache_seqlens is None:
|
| 294 |
+
klens = torch.full((B,), k_cache.shape[1], device=q.device, dtype=torch.long)
|
| 295 |
+
elif isinstance(cache_seqlens, int):
|
| 296 |
+
klens = torch.full((B,), cache_seqlens + appended, device=q.device, dtype=torch.long)
|
| 297 |
+
else:
|
| 298 |
+
klens = cache_seqlens.to(q.device, torch.long) + appended
|
| 299 |
+
|
| 300 |
+
uniform = bool((klens == klens[0]).all().item())
|
| 301 |
+
|
| 302 |
+
# ---- fast path: contiguous cache, uniform length -> dense kernel ----
|
| 303 |
+
if not paged and uniform:
|
| 304 |
+
L = int(klens[0].item())
|
| 305 |
+
return flash_attn_func(
|
| 306 |
+
q, k_cache[:, :L], v_cache[:, :L],
|
| 307 |
+
softmax_scale=softmax_scale, causal=causal,
|
| 308 |
+
window_size=(window_size_left, window_size_right, sink_size),
|
| 309 |
+
s_aux=s_aux, return_lse=return_lse,
|
| 310 |
+
)
|
| 311 |
+
|
| 312 |
+
# ---- general path: variable cache lengths -> split-KV varlen kernel ----
|
| 313 |
+
cu_q = torch.arange(0, (B + 1) * Sq, Sq, device=q.device, dtype=torch.int32)
|
| 314 |
+
cu_k = torch.zeros(B + 1, device=q.device, dtype=torch.int32)
|
| 315 |
+
cu_k[1:] = torch.cumsum(klens, 0).to(torch.int32)
|
| 316 |
+
q_packed = q.reshape(B * Sq, Hq, D)
|
| 317 |
+
if paged:
|
| 318 |
+
k_packed, v_packed = k_cache, v_cache # kernel gathers via block_table
|
| 319 |
+
else:
|
| 320 |
+
k_packed = torch.cat([k_cache[b, : int(klens[b])] for b in range(B)], dim=0)
|
| 321 |
+
v_packed = torch.cat([v_cache[b, : int(klens[b])] for b in range(B)], dim=0)
|
| 322 |
+
|
| 323 |
+
out = flash_attn_varlen_func(
|
| 324 |
+
q_packed, k_packed, v_packed, cu_q, cu_k, Sq, int(klens.max().item()),
|
| 325 |
+
softmax_scale=softmax_scale, causal=causal,
|
| 326 |
+
window_size=(window_size_left, window_size_right, sink_size),
|
| 327 |
+
block_table=block_table, s_aux=s_aux, return_lse=return_lse,
|
| 328 |
+
)
|
| 329 |
+
if return_lse:
|
| 330 |
+
out, lse = out
|
| 331 |
+
return out.reshape(B, Sq, Hq, D), lse
|
| 332 |
+
return out.reshape(B, Sq, Hq, D)
|
build/torch212-rocm72-x86_64-linux/symbols.json
ADDED
|
@@ -0,0 +1,349 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": 1,
|
| 3 |
+
"module": "_aiter_flash_attn_ck_rocm_fe2bb66",
|
| 4 |
+
"functions": [
|
| 5 |
+
{
|
| 6 |
+
"name": "flash_attn_func",
|
| 7 |
+
"qualname": "flash_attn_func",
|
| 8 |
+
"module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha",
|
| 9 |
+
"kind": "function",
|
| 10 |
+
"doc": "Dense FlashAttention forward.\n\n``q, k, v`` have shape ``(batch, seqlen, nheads, headdim)``. ``window_size``\nis ``(left, right, sink_size)``; ``s_aux`` is an optional ``(nheads_q,)``\nfp32 tensor of learnable sink logits. Extra keyword arguments (e.g. those\npassed by the transformers flash-attention interface) are ignored.",
|
| 11 |
+
"signature": {
|
| 12 |
+
"parameters": [
|
| 13 |
+
{
|
| 14 |
+
"name": "q",
|
| 15 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 16 |
+
"default": null,
|
| 17 |
+
"annotation": "torch.Tensor"
|
| 18 |
+
},
|
| 19 |
+
{
|
| 20 |
+
"name": "k",
|
| 21 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 22 |
+
"default": null,
|
| 23 |
+
"annotation": "torch.Tensor"
|
| 24 |
+
},
|
| 25 |
+
{
|
| 26 |
+
"name": "v",
|
| 27 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 28 |
+
"default": null,
|
| 29 |
+
"annotation": "torch.Tensor"
|
| 30 |
+
},
|
| 31 |
+
{
|
| 32 |
+
"name": "dropout_p",
|
| 33 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 34 |
+
"default": "0.0",
|
| 35 |
+
"annotation": "float"
|
| 36 |
+
},
|
| 37 |
+
{
|
| 38 |
+
"name": "softmax_scale",
|
| 39 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 40 |
+
"default": "None",
|
| 41 |
+
"annotation": "float | None"
|
| 42 |
+
},
|
| 43 |
+
{
|
| 44 |
+
"name": "causal",
|
| 45 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 46 |
+
"default": "False",
|
| 47 |
+
"annotation": "bool"
|
| 48 |
+
},
|
| 49 |
+
{
|
| 50 |
+
"name": "window_size",
|
| 51 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 52 |
+
"default": "(-1, -1, 0)",
|
| 53 |
+
"annotation": "Tuple[int, ...]"
|
| 54 |
+
},
|
| 55 |
+
{
|
| 56 |
+
"name": "bias",
|
| 57 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 58 |
+
"default": "None",
|
| 59 |
+
"annotation": "torch.Tensor | None"
|
| 60 |
+
},
|
| 61 |
+
{
|
| 62 |
+
"name": "alibi_slopes",
|
| 63 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 64 |
+
"default": "None",
|
| 65 |
+
"annotation": "torch.Tensor | None"
|
| 66 |
+
},
|
| 67 |
+
{
|
| 68 |
+
"name": "return_lse",
|
| 69 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 70 |
+
"default": "False",
|
| 71 |
+
"annotation": "bool"
|
| 72 |
+
},
|
| 73 |
+
{
|
| 74 |
+
"name": "return_attn_probs",
|
| 75 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 76 |
+
"default": "False",
|
| 77 |
+
"annotation": "bool"
|
| 78 |
+
},
|
| 79 |
+
{
|
| 80 |
+
"name": "s_aux",
|
| 81 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 82 |
+
"default": "None",
|
| 83 |
+
"annotation": "torch.Tensor | None"
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"name": "kwargs",
|
| 87 |
+
"kind": "VAR_KEYWORD",
|
| 88 |
+
"default": null,
|
| 89 |
+
"annotation": null
|
| 90 |
+
}
|
| 91 |
+
],
|
| 92 |
+
"return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]"
|
| 93 |
+
}
|
| 94 |
+
},
|
| 95 |
+
{
|
| 96 |
+
"name": "flash_attn_varlen_func",
|
| 97 |
+
"qualname": "flash_attn_varlen_func",
|
| 98 |
+
"module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha",
|
| 99 |
+
"kind": "function",
|
| 100 |
+
"doc": "Variable-length / packed FlashAttention forward.\n\n``q`` has shape ``(total_q, nheads, headdim)``; ``cu_seqlens_*`` are int32\ncumulative offsets of shape ``(batch + 1,)``. Sinks are supported through\n``window_size[2]`` / ``s_aux`` as in :func:`flash_attn_func`. Extra keyword\narguments (e.g. from the transformers flash-attention interface) are ignored.",
|
| 101 |
+
"signature": {
|
| 102 |
+
"parameters": [
|
| 103 |
+
{
|
| 104 |
+
"name": "q",
|
| 105 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 106 |
+
"default": null,
|
| 107 |
+
"annotation": "torch.Tensor"
|
| 108 |
+
},
|
| 109 |
+
{
|
| 110 |
+
"name": "k",
|
| 111 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 112 |
+
"default": null,
|
| 113 |
+
"annotation": "torch.Tensor"
|
| 114 |
+
},
|
| 115 |
+
{
|
| 116 |
+
"name": "v",
|
| 117 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 118 |
+
"default": null,
|
| 119 |
+
"annotation": "torch.Tensor"
|
| 120 |
+
},
|
| 121 |
+
{
|
| 122 |
+
"name": "cu_seqlens_q",
|
| 123 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 124 |
+
"default": null,
|
| 125 |
+
"annotation": "torch.Tensor"
|
| 126 |
+
},
|
| 127 |
+
{
|
| 128 |
+
"name": "cu_seqlens_k",
|
| 129 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 130 |
+
"default": null,
|
| 131 |
+
"annotation": "torch.Tensor"
|
| 132 |
+
},
|
| 133 |
+
{
|
| 134 |
+
"name": "max_seqlen_q",
|
| 135 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 136 |
+
"default": null,
|
| 137 |
+
"annotation": "int"
|
| 138 |
+
},
|
| 139 |
+
{
|
| 140 |
+
"name": "max_seqlen_k",
|
| 141 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 142 |
+
"default": null,
|
| 143 |
+
"annotation": "int"
|
| 144 |
+
},
|
| 145 |
+
{
|
| 146 |
+
"name": "dropout_p",
|
| 147 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 148 |
+
"default": "0.0",
|
| 149 |
+
"annotation": "float"
|
| 150 |
+
},
|
| 151 |
+
{
|
| 152 |
+
"name": "softmax_scale",
|
| 153 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 154 |
+
"default": "None",
|
| 155 |
+
"annotation": "float | None"
|
| 156 |
+
},
|
| 157 |
+
{
|
| 158 |
+
"name": "causal",
|
| 159 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 160 |
+
"default": "False",
|
| 161 |
+
"annotation": "bool"
|
| 162 |
+
},
|
| 163 |
+
{
|
| 164 |
+
"name": "window_size",
|
| 165 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 166 |
+
"default": "(-1, -1, 0)",
|
| 167 |
+
"annotation": "Tuple[int, ...]"
|
| 168 |
+
},
|
| 169 |
+
{
|
| 170 |
+
"name": "bias",
|
| 171 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 172 |
+
"default": "None",
|
| 173 |
+
"annotation": "torch.Tensor | None"
|
| 174 |
+
},
|
| 175 |
+
{
|
| 176 |
+
"name": "alibi_slopes",
|
| 177 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 178 |
+
"default": "None",
|
| 179 |
+
"annotation": "torch.Tensor | None"
|
| 180 |
+
},
|
| 181 |
+
{
|
| 182 |
+
"name": "return_lse",
|
| 183 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 184 |
+
"default": "False",
|
| 185 |
+
"annotation": "bool"
|
| 186 |
+
},
|
| 187 |
+
{
|
| 188 |
+
"name": "return_attn_probs",
|
| 189 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 190 |
+
"default": "False",
|
| 191 |
+
"annotation": "bool"
|
| 192 |
+
},
|
| 193 |
+
{
|
| 194 |
+
"name": "block_table",
|
| 195 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 196 |
+
"default": "None",
|
| 197 |
+
"annotation": "torch.Tensor | None"
|
| 198 |
+
},
|
| 199 |
+
{
|
| 200 |
+
"name": "min_seqlen_q",
|
| 201 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 202 |
+
"default": "0",
|
| 203 |
+
"annotation": "int"
|
| 204 |
+
},
|
| 205 |
+
{
|
| 206 |
+
"name": "logits_soft_cap",
|
| 207 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 208 |
+
"default": "0.0",
|
| 209 |
+
"annotation": "float"
|
| 210 |
+
},
|
| 211 |
+
{
|
| 212 |
+
"name": "zero_tensors",
|
| 213 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 214 |
+
"default": "False",
|
| 215 |
+
"annotation": "bool"
|
| 216 |
+
},
|
| 217 |
+
{
|
| 218 |
+
"name": "cu_seqlens_q_padded",
|
| 219 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 220 |
+
"default": "None",
|
| 221 |
+
"annotation": "torch.Tensor | None"
|
| 222 |
+
},
|
| 223 |
+
{
|
| 224 |
+
"name": "cu_seqlens_k_padded",
|
| 225 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 226 |
+
"default": "None",
|
| 227 |
+
"annotation": "torch.Tensor | None"
|
| 228 |
+
},
|
| 229 |
+
{
|
| 230 |
+
"name": "s_aux",
|
| 231 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 232 |
+
"default": "None",
|
| 233 |
+
"annotation": "torch.Tensor | None"
|
| 234 |
+
},
|
| 235 |
+
{
|
| 236 |
+
"name": "kwargs",
|
| 237 |
+
"kind": "VAR_KEYWORD",
|
| 238 |
+
"default": null,
|
| 239 |
+
"annotation": null
|
| 240 |
+
}
|
| 241 |
+
],
|
| 242 |
+
"return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]"
|
| 243 |
+
}
|
| 244 |
+
},
|
| 245 |
+
{
|
| 246 |
+
"name": "flash_attn_with_kvcache",
|
| 247 |
+
"qualname": "flash_attn_with_kvcache",
|
| 248 |
+
"module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha",
|
| 249 |
+
"kind": "function",
|
| 250 |
+
"doc": "FlashAttention against a KV cache, built on the CK split-KV kernel.\n\n``q`` is ``(batch, seqlen_q, nheads, headdim)``. For a contiguous cache,\n``k_cache``/``v_cache`` are ``(batch, seqlen_cache, nheads_k, headdim)`` and\n``cache_seqlens`` (int or ``(batch,)``) is the number of valid cache entries\n*before* appending. If ``k``/``v`` (the new tokens) are given, they are\nwritten into the cache in place. The query attends to the valid prefix\n(``cache_seqlens`` + ``seqlen_q``) with bottom-right causal alignment.\n\nNotes / current limitations:\n- Rotary embedding is not applied here; pass ``q``/``k`` already rotated\n (``rotary_cos``/``rotary_sin`` are rejected).\n- Paged caches are addressed via ``block_table`` (forwarded to the split-KV\n varlen kernel); in-cache append for paged layouts must be done by the\n caller (pass ``k=v=None``).",
|
| 251 |
+
"signature": {
|
| 252 |
+
"parameters": [
|
| 253 |
+
{
|
| 254 |
+
"name": "q",
|
| 255 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 256 |
+
"default": null,
|
| 257 |
+
"annotation": "torch.Tensor"
|
| 258 |
+
},
|
| 259 |
+
{
|
| 260 |
+
"name": "k_cache",
|
| 261 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 262 |
+
"default": null,
|
| 263 |
+
"annotation": "torch.Tensor"
|
| 264 |
+
},
|
| 265 |
+
{
|
| 266 |
+
"name": "v_cache",
|
| 267 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 268 |
+
"default": null,
|
| 269 |
+
"annotation": "torch.Tensor"
|
| 270 |
+
},
|
| 271 |
+
{
|
| 272 |
+
"name": "k",
|
| 273 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 274 |
+
"default": "None",
|
| 275 |
+
"annotation": "torch.Tensor | None"
|
| 276 |
+
},
|
| 277 |
+
{
|
| 278 |
+
"name": "v",
|
| 279 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 280 |
+
"default": "None",
|
| 281 |
+
"annotation": "torch.Tensor | None"
|
| 282 |
+
},
|
| 283 |
+
{
|
| 284 |
+
"name": "rotary_cos",
|
| 285 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 286 |
+
"default": "None",
|
| 287 |
+
"annotation": "torch.Tensor | None"
|
| 288 |
+
},
|
| 289 |
+
{
|
| 290 |
+
"name": "rotary_sin",
|
| 291 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 292 |
+
"default": "None",
|
| 293 |
+
"annotation": "torch.Tensor | None"
|
| 294 |
+
},
|
| 295 |
+
{
|
| 296 |
+
"name": "cache_seqlens",
|
| 297 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 298 |
+
"default": "None",
|
| 299 |
+
"annotation": null
|
| 300 |
+
},
|
| 301 |
+
{
|
| 302 |
+
"name": "block_table",
|
| 303 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 304 |
+
"default": "None",
|
| 305 |
+
"annotation": "torch.Tensor | None"
|
| 306 |
+
},
|
| 307 |
+
{
|
| 308 |
+
"name": "softmax_scale",
|
| 309 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 310 |
+
"default": "None",
|
| 311 |
+
"annotation": "float | None"
|
| 312 |
+
},
|
| 313 |
+
{
|
| 314 |
+
"name": "causal",
|
| 315 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 316 |
+
"default": "True",
|
| 317 |
+
"annotation": "bool"
|
| 318 |
+
},
|
| 319 |
+
{
|
| 320 |
+
"name": "window_size",
|
| 321 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 322 |
+
"default": "(-1, -1, 0)",
|
| 323 |
+
"annotation": "Tuple[int, ...]"
|
| 324 |
+
},
|
| 325 |
+
{
|
| 326 |
+
"name": "s_aux",
|
| 327 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 328 |
+
"default": "None",
|
| 329 |
+
"annotation": "torch.Tensor | None"
|
| 330 |
+
},
|
| 331 |
+
{
|
| 332 |
+
"name": "return_lse",
|
| 333 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 334 |
+
"default": "False",
|
| 335 |
+
"annotation": "bool"
|
| 336 |
+
},
|
| 337 |
+
{
|
| 338 |
+
"name": "kwargs",
|
| 339 |
+
"kind": "VAR_KEYWORD",
|
| 340 |
+
"default": null,
|
| 341 |
+
"annotation": null
|
| 342 |
+
}
|
| 343 |
+
],
|
| 344 |
+
"return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]"
|
| 345 |
+
}
|
| 346 |
+
}
|
| 347 |
+
],
|
| 348 |
+
"layers": []
|
| 349 |
+
}
|
build/torch213-rocm71-x86_64-linux/__init__.py
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""AITER Composable-Kernel FlashAttention kernels for AMD ROCm (compiled HIP).
|
| 2 |
+
|
| 3 |
+
Compiled (Composable Kernel / HIP) counterpart to the Triton-based
|
| 4 |
+
``aiter-flash-attn`` package, repackaged from the CK FMHA forward path of the
|
| 5 |
+
`ROCm/aiter <https://github.com/ROCm/aiter>`_ project as a self-contained
|
| 6 |
+
Hugging Face Hub kernel. Exposes the FlashAttention forward entry points with
|
| 7 |
+
learnable attention sink support (e.g. gpt-oss).
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
from .mha import (
|
| 11 |
+
flash_attn_func,
|
| 12 |
+
flash_attn_varlen_func,
|
| 13 |
+
flash_attn_with_kvcache,
|
| 14 |
+
)
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
__kernel_metadata__ = {
|
| 18 |
+
"license": "mit",
|
| 19 |
+
}
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
__all__ = [
|
| 23 |
+
"__kernel_metadata__",
|
| 24 |
+
"flash_attn_func",
|
| 25 |
+
"flash_attn_varlen_func",
|
| 26 |
+
"flash_attn_with_kvcache",
|
| 27 |
+
]
|
build/torch213-rocm71-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:2a171f33a155682ab9672636dfb2f60fa988da30f3d8a9f6fff3d48fb6032ea6
|
| 3 |
+
size 201962776
|
build/torch213-rocm71-x86_64-linux/_ops.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
from . import _aiter_flash_attn_ck_rocm_fe2bb66
|
| 3 |
+
ops = torch.ops._aiter_flash_attn_ck_rocm_fe2bb66
|
| 4 |
+
|
| 5 |
+
def add_op_namespace_prefix(op_name: str):
|
| 6 |
+
"""
|
| 7 |
+
Prefix op by namespace.
|
| 8 |
+
"""
|
| 9 |
+
return f"_aiter_flash_attn_ck_rocm_fe2bb66::{op_name}"
|
build/torch213-rocm71-x86_64-linux/metadata.json
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "aiter-flash-attn-ck",
|
| 3 |
+
"id": "_aiter_flash_attn_ck_rocm_fe2bb66",
|
| 4 |
+
"version": 1,
|
| 5 |
+
"kernels-minver": "0.14.0",
|
| 6 |
+
"license": "MIT",
|
| 7 |
+
"python-depends": [],
|
| 8 |
+
"kernel-depends": [],
|
| 9 |
+
"backend": {
|
| 10 |
+
"type": "rocm",
|
| 11 |
+
"archs": [
|
| 12 |
+
"gfx942",
|
| 13 |
+
"gfx950"
|
| 14 |
+
]
|
| 15 |
+
},
|
| 16 |
+
"digest": {
|
| 17 |
+
"algorithm": "sha256",
|
| 18 |
+
"files": {
|
| 19 |
+
"__init__.py": "7wj1GTm2bqTLLCyT4xLuAk/+JWYrhINjY/ODMNWhgZ0=",
|
| 20 |
+
"_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so": "KhcfM6FVaCq5ZyY237L2D6mI2jDz2Kn2//PUj7YDLqY=",
|
| 21 |
+
"_ops.py": "4WkdYDWWJp60zep+oW/5ym9vIeVMNfrgRCkP3o/w7yU=",
|
| 22 |
+
"mha.py": "zCAjY4O+DiCzmEL+W9sqXxZaqjsj4RJbWFcyPpUZejk=",
|
| 23 |
+
"symbols.json": "rhnF4H8Jsb6I+YPxIVFMgnLcTQJODEXbZ7ge77NZXAU="
|
| 24 |
+
}
|
| 25 |
+
},
|
| 26 |
+
"provenance": {
|
| 27 |
+
"kernel-builder": {
|
| 28 |
+
"version": "0.18.0-dev0",
|
| 29 |
+
"commit": "d054906a13710b811b96697b4635f03020842733",
|
| 30 |
+
"dirty": false
|
| 31 |
+
},
|
| 32 |
+
"kernel": {
|
| 33 |
+
"commit": "fe2bb66fee6c2a77eccf08a0508dc57428d488ed",
|
| 34 |
+
"dirty": false
|
| 35 |
+
}
|
| 36 |
+
}
|
| 37 |
+
}
|
build/torch213-rocm71-x86_64-linux/metadata.json.sigstore
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json", "verificationMaterial":{"certificate":{"rawBytes":"MIIHLTCCBrKgAwIBAgIUYhDBz3s6J742wscabPTQJuZF0E4wCgYIKoZIzj0EAwMwNzEVMBMGA1UEChMMc2lnc3RvcmUuZGV2MR4wHAYDVQQDExVzaWdzdG9yZS1pbnRlcm1lZGlhdGUwHhcNMjYxMDE5MjIzMTE4WhcNMjYxMDE5MjI0MTE4WjAAMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEp3g/dI77cg1tnq5eu54MWYrJ8cbb8SzEVubcy/U67+Aiwr4Y0MWc384cRXpKZj0FhdoGxrrETLe96k9d+zBzgaOCBdEwggXNMA4GA1UdDwEB/wQEAwIHgDATBgNVHSUEDDAKBggrBgEFBQcDAzAfBgNVHSMEGDAWgBTf0+nPViQRlvmo2OkoVaLGLhhkPzBrBgNVHREBAf8EYTBfhl1odHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkvLmdpdGh1Yi93b3JrZmxvd3MvYnVpbGQueWFtbEByZWZzL2hlYWRzL21haW4wOQYKKwYBBAGDvzABAQQraHR0cHM6Ly90b2tlbi5hY3Rpb25zLmdpdGh1YnVzZXJjb250ZW50LmNvbTAfBgorBgEEAYO/MAECBBF3b3JrZmxvd19kaXNwYXRjaDA2BgorBgEEAYO/MAEDBChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMBMGCisGAQQBg78wAQQEBUJ1aWxkMCsGCisGAQQBg78wAQUEHWh1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5MB0GCisGAQQBg78wAQYED3JlZnMvaGVhZHMvbWFpbjA7BgorBgEEAYO/MAEIBC0MK2h0dHBzOi8vdG9rZW4uYWN0aW9ucy5naXRodWJ1c2VyY29udGVudC5jb20wbQYKKwYBBAGDvzABCQRfDF1odHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkvLmdpdGh1Yi93b3JrZmxvd3MvYnVpbGQueWFtbEByZWZzL2hlYWRzL21haW4wOAYKKwYBBAGDvzABCgQqDChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMBsGCisGAQQBg78wAQsEDQwLc2VsZi1ob3N0ZWQwQAYKKwYBBAGDvzABDAQyDDBodHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkwOAYKKwYBBAGDvzABDQQqDChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMB8GCisGAQQBg78wAQ4EEQwPcmVmcy9oZWFkcy9tYWluMBoGCisGAQQBg78wAQ8EDAwKMTA3MTQ3NTUyOTAuBgorBgEEAYO/MAEQBCAMHmh0dHBzOi8vZ2l0aHViLmNvbS9odWdnaW5nZmFjZTAYBgorBgEEAYO/MAERBAoMCDI1NzIwNzQzMG0GCisGAQQBg78wARIEXwxdaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5Ly5naXRodWIvd29ya2Zsb3dzL2J1aWxkLnlhbWxAcmVmcy9oZWFkcy9tYWluMDgGCisGAQQBg78wARMEKgwoZmUyYmI2NmZlZTZjMmE3N2VjY2YwOGEwNTA4ZGM1NzQyOGQ0ODhlZDAhBgorBgEEAYO/MAEUBBMMEXdvcmtmbG93X2Rpc3BhdGNoMGQGCisGAQQBg78wARUEVgxUaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5L2FjdGlvbnMvcnVucy8zOTIzOTUxODEwNC9hdHRlbXB0cy8xMBYGCisGAQQBg78wARYECAwGcHVibGljMEYGCisGAQQBg78wARgEOAw2cmVwbzpodWdnaW5nZmFjZS9rZXJuZWxzLWNvbW11bml0eTpyZWY6cmVmcy9oZWFkcy9tYWluMIGKBgorBgEEAdZ5AgQCBHwEegB4AHYA3T0wasbHETJjGR4cmWc3AqJKXrjePK3/h4pygC8p7o4AAAGhVkplLwAABAMARzBFAiAYzcmarOFIpCdMHLsWMMcmiX3uiDqTtrSbhnugLl9KbQIhAJimKU6Gp6Muq/12zl25mMiXo7YzY9oPcsSkZyGlyJZwMAoGCCqGSM49BAMDA2kAMGYCMQD8NasCHiPCLAtolVO5/0rl5UfBymvjywi0NVKAJdcAvP61/IJlRbX5IZbatI+8YOsCMQCGhXN2BT6CTuaWC3qwLFzACyTV4uinQUnHOzEDc74z1+uGTmHfVTRDIi2/tFQuuNM="}, "tlogEntries":[{"logIndex":"3327212209", "logId":{"keyId":"wNI9atQGlz+VWfO6LRygH4QUfY/8W4RFwiT5i5WRgB0="}, "kindVersion":{"kind":"hashedrekord", "version":"0.0.1"}, "integratedTime":"1792449078", "inclusionPromise":{"signedEntryTimestamp":"MEQCIBsAQmhJQzBVzalYjG0DrFcgZBeNwlfSjSgAmWKG0sPLAiAg/J0Hzy6ImkWvWbDx3/jTT3+cRJDBm2PIrIDTQTwEuw=="}, "inclusionProof":{"logIndex":"3205307947", "rootHash":"W+fxNQGQRlQR8NP5epr4cduN8DmMRMB26mm1Ob1vTrs=", "treeSize":"3205307963", "hashes":["S7xwjCFRfoN65j+YWkYcaKfA6xnjEY+JNyw73H3LYfg=", "1ZB+altVOQMte65LrxRL5bTxYbBz43joc2eTBAALLpA=", "kBfrXfa6yvdyCvkB8KxtuAweraT6oqFH4L025R6tbjk=", "5AYjk/jc91WxLoeKt6AMwJV3depI3V6Yi1pPZMjQMJw=", "iFo9RWcriE709SPAUnZ3E60KJTBscfs+3bxe9WWL2OA=", "pe8st8CnUIruORVX9JaZ2kalqSjxb7oo2wfilP83+bk=", "gBGf/lDkk/5tDR8CLTuE5Pt2YTL1zjLs12T3vvxq7X4=", "0aSE/O6jCq82vSwoybvpblRnnP72+1gEInKy6Wyc3E0=", "5egUlZ4Pi0Ix/qLyCDXocsWywJCHqrde/S06vOZvlOM=", "QsXufUpoKtSjKAB19K1G00umQpWvPHlBNuRGFiiVjQQ=", "QIMkwQbwAewhLJd//5FFwI3SnyNQBGjCgJrelfG8vhc=", "Et17CACUv9QN64irUJuBjsrgRGkGnnJ9xacRV/aBdwQ=", "lqgEwkxhDMs/gdIAE4lmjfMwg7nDmv7lda6O0LGliCw=", "HovTPBWzoErvUg1u5sIi3x1ZonZ7yJekVsBrmpomCIA=", "G+nk7mgE5h6dUmv2UY7NDJrY3WvrZY31raaO1AETt+o=", "qlTWNDzERnTBQVcHHXXVPjiUP0SsikGxfcxVlgqVFDo=", "gg9GNApNke3+zNhl4tsWtmIycCFrOMCB3lys6XuSQNQ=", "NlOclldmKPA1tMKIskqHAwuQDYHp31YwpK+OOMrMccc=", "qxzHanAzz57SDdmJe0B7bJK72NTIBbwEMGBBKvDOROw=", "xH/DCseLHr9eKoYT8qsORZK7zVdEGYWHuVtsVrD95wY="], "checkpoint":{"envelope":"rekor.sigstore.dev - 1193050959916656506\n3205307963\nW+fxNQGQRlQR8NP5epr4cduN8DmMRMB26mm1Ob1vTrs=\n\n— rekor.sigstore.dev wNI9ajBFAiEAxuRsGa2QG9aIT9+2qqvRf3gfyS1DbDaXilVScnJohYUCIDNWE9xwg2HhHzFKbgG8xMK0LNtqkdJC9u2OLYFMDY6H\n"}}, "canonicalizedBody":"eyJhcGlWZXJzaW9uIjoiMC4wLjEiLCJraW5kIjoiaGFzaGVkcmVrb3JkIiwic3BlYyI6eyJkYXRhIjp7Imhhc2giOnsiYWxnb3JpdGhtIjoic2hhMjU2IiwidmFsdWUiOiIwMDhiYTkyYTNlY2M5MjA5ZDBiMDUxOTllZGEzY2IxYmE0ZWM4OGMzZTVmOGI0YWZhYzAwZmIyMTk4OTJkYjRjIn19LCJzaWduYXR1cmUiOnsiY29udGVudCI6Ik1FUUNIMXBXbzdsRVJEbXhFZlk3emZnMlBscjJTb2xqZWR1azZCZGhqYU1samZBQ0lRQzFaZks4Y3VFVERodW5rT0R0MTg3TTJwdlRpOEVWczg0WEwvNDY3RGtXbVE9PSIsInB1YmxpY0tleSI6eyJjb250ZW50IjoiTFMwdExTMUNSVWRKVGlCRFJWSlVTVVpKUTBGVVJTMHRMUzB0Q2sxSlNVaE1WRU5EUW5KTFowRjNTVUpCWjBsVldXaEVRbm96Y3paS056UXlkM05qWVdKUVZGRktkVnBHTUVVMGQwTm5XVWxMYjFwSmVtb3dSVUYzVFhjS1RucEZWazFDVFVkQk1WVkZRMmhOVFdNeWJHNWpNMUoyWTIxVmRWcEhWakpOVWpSM1NFRlpSRlpSVVVSRmVGWjZZVmRrZW1SSE9YbGFVekZ3WW01U2JBcGpiVEZzV2tkc2FHUkhWWGRJYUdOT1RXcFplRTFFUlRWTmFrbDZUVlJGTkZkb1kwNU5hbGw0VFVSRk5VMXFTVEJOVkVVMFYycEJRVTFHYTNkRmQxbElDa3R2V2tsNmFqQkRRVkZaU1V0dldrbDZhakJFUVZGalJGRm5RVVZ3TTJjdlpFazNOMk5uTVhSdWNUVmxkVFUwVFZkWmNrbzRZMkppT0ZONlJWWjFZbU1LZVM5Vk5qY3JRV2wzY2pSWk1FMVhZek00TkdOU1dIQkxXbW93Um1oa2IwZDRjbkpGVkV4bE9UWnJPV1FyZWtKNloyRlBRMEprUlhkbloxaE9UVUUwUndwQk1WVmtSSGRGUWk5M1VVVkJkMGxJWjBSQlZFSm5UbFpJVTFWRlJFUkJTMEpuWjNKQ1owVkdRbEZqUkVGNlFXWkNaMDVXU0ZOTlJVZEVRVmRuUWxSbUNqQXJibEJXYVZGU2JIWnRiekpQYTI5V1lVeEhUR2hvYTFCNlFuSkNaMDVXU0ZKRlFrRm1PRVZaVkVKbWFHd3hiMlJJVW5kamVtOTJUREprY0dSSGFERUtXV2sxYW1JeU1IWmhTRlp1V2pKc2RWb3lXbWhaTWxWMllUSldlV0p0Vm5OamVURnFZakl4ZEdSWE5YQmtTR3QyVEcxa2NHUkhhREZaYVRrellqTktjZ3BhYlhoMlpETk5kbGx1Vm5CaVIxRjFaVmRHZEdKRlFubGFWMXA2VERKb2JGbFhVbnBNTWpGb1lWYzBkMDlSV1V0TGQxbENRa0ZIUkhaNlFVSkJVVkZ5Q21GSVVqQmpTRTAyVEhrNU1HSXlkR3hpYVRWb1dUTlNjR0l5TlhwTWJXUndaRWRvTVZsdVZucGFXRXBxWWpJMU1GcFhOVEJNYlU1MllsUkJaa0puYjNJS1FtZEZSVUZaVHk5TlFVVkRRa0pHTTJJelNuSmFiWGgyWkRFNWEyRllUbmRaV0ZKcVlVUkJNa0puYjNKQ1owVkZRVmxQTDAxQlJVUkNRMmh0V2xSS2FRcFphbGt5V20xV2JFNXRUWGxaVkdNeldsZE9hbHBxUVRSWlZFRXhUVVJvYTFsNlZUTk9SRWswV2tSUk5FOUhWbXROUWsxSFEybHpSMEZSVVVKbk56aDNDa0ZSVVVWQ1ZVb3hZVmQ0YTAxRGMwZERhWE5IUVZGUlFtYzNPSGRCVVZWRlNGZG9NVm95WkhCaWJXUnRXVmRPYkV3eWRHeGpiVFZzWWtoTmRGa3lPWFFLWWxoV2RXRllValZOUWpCSFEybHpSMEZSVVVKbk56aDNRVkZaUlVRelNteGFiazEyWVVkV2FGcElUWFppVjBad1ltcEJOMEpuYjNKQ1owVkZRVmxQTHdwTlFVVkpRa013VFVzeWFEQmtTRUo2VDJrNGRtUkhPWEphVnpSMVdWZE9NR0ZYT1hWamVUVnVZVmhTYjJSWFNqRmpNbFo1V1RJNWRXUkhWblZrUXpWcUNtSXlNSGRpVVZsTFMzZFpRa0pCUjBSMmVrRkNRMUZTWmtSR01XOWtTRkozWTNwdmRrd3laSEJrUjJneFdXazFhbUl5TUhaaFNGWnVXakpzZFZveVdtZ0tXVEpWZG1FeVZubGliVlp6WTNreGFtSXlNWFJrVnpWd1pFaHJka3h0WkhCa1IyZ3hXV2s1TTJJelNuSmFiWGgyWkROTmRsbHVWbkJpUjFGMVpWZEdkQXBpUlVKNVdsZGFla3d5YUd4WlYxSjZUREl4YUdGWE5IZFBRVmxMUzNkWlFrSkJSMFIyZWtGQ1EyZFJjVVJEYUcxYVZFcHBXV3BaTWxwdFZteE9iVTE1Q2xsVVl6TmFWMDVxV21wQk5GbFVRVEZOUkdocldYcFZNMDVFU1RSYVJGRTBUMGRXYTAxQ2MwZERhWE5IUVZGUlFtYzNPSGRCVVhORlJGRjNUR015Vm5NS1dta3hiMkl6VGpCYVYxRjNVVUZaUzB0M1dVSkNRVWRFZG5wQlFrUkJVWGxFUkVKdlpFaFNkMk42YjNaTU1tUndaRWRvTVZscE5XcGlNakIyWVVoV2JncGFNbXgxV2pKYWFGa3lWWFpoTWxaNVltMVdjMk41TVdwaU1qRjBaRmMxY0dSSWEzZFBRVmxMUzNkWlFrSkJSMFIyZWtGQ1JGRlJjVVJEYUcxYVZFcHBDbGxxV1RKYWJWWnNUbTFOZVZsVVl6TmFWMDVxV21wQk5GbFVRVEZOUkdocldYcFZNMDVFU1RSYVJGRTBUMGRXYTAxQ09FZERhWE5IUVZGUlFtYzNPSGNLUVZFMFJVVlJkMUJqYlZadFkzazViMXBYUm10amVUbDBXVmRzZFUxQ2IwZERhWE5IUVZGUlFtYzNPSGRCVVRoRlJFRjNTMDFVUVROTlZGRXpUbFJWZVFwUFZFRjFRbWR2Y2tKblJVVkJXVTh2VFVGRlVVSkRRVTFJYldnd1pFaENlazlwT0haYU1td3dZVWhXYVV4dFRuWmlVemx2WkZka2JtRlhOVzVhYlVacUNscFVRVmxDWjI5eVFtZEZSVUZaVHk5TlFVVlNRa0Z2VFVORVNURk9la2wzVG5wUmVrMUhNRWREYVhOSFFWRlJRbWMzT0hkQlVrbEZXSGQ0WkdGSVVqQUtZMGhOTmt4NU9XNWhXRkp2WkZkSmRWa3lPWFJNTW1neFdqSmtjR0p0WkcxWlYwNXNUREowYkdOdE5XeGlTRTEwV1RJNWRHSllWblZoV0ZJMVRIazFiZ3BoV0ZKdlpGZEpkbVF5T1hsaE1scHpZak5rZWt3eVNqRmhWM2hyVEc1c2FHSlhlRUZqYlZadFkzazViMXBYUm10amVUbDBXVmRzZFUxRVowZERhWE5IQ2tGUlVVSm5OemgzUVZKTlJVdG5kMjlhYlZWNVdXMUpNazV0V214YVZGcHFUVzFGTTA0eVZtcFpNbGwzVDBkRmQwNVVRVFJhUjAweFRucFJlVTlIVVRBS1QwUm9iRnBFUVdoQ1oyOXlRbWRGUlVGWlR5OU5RVVZWUWtKTlRVVllaSFpqYlhSdFlrYzVNMWd5VW5Cak0wSm9aRWRPYjAxSFVVZERhWE5IUVZGUlFncG5OemgzUVZKVlJWWm5lRlZoU0ZJd1kwaE5Oa3g1T1c1aFdGSnZaRmRKZFZreU9YUk1NbWd4V2pKa2NHSnRaRzFaVjA1c1RESjBiR050Tld4aVNFMTBDbGt5T1hSaVdGWjFZVmhTTlV3eVJtcGtSMngyWW01TmRtTnVWblZqZVRoNlQxUkplazlVVlhoUFJFVjNUa001YUdSSVVteGlXRUl3WTNrNGVFMUNXVWNLUTJselIwRlJVVUpuTnpoM1FWSlpSVU5CZDBkalNGWnBZa2RzYWsxRldVZERhWE5IUVZGUlFtYzNPSGRCVW1kRlQwRjNNbU50Vm5kaWVuQnZaRmRrYmdwaFZ6VnVXbTFHYWxwVE9YSmFXRXAxV2xkNGVreFhUblppVnpFeFltMXNNR1ZVY0hsYVYxazJZMjFXYldONU9XOWFWMFpyWTNrNWRGbFhiSFZOU1VkTENrSm5iM0pDWjBWRlFXUmFOVUZuVVVOQ1NIZEZaV2RDTkVGSVdVRXpWREIzWVhOaVNFVlVTbXBIVWpSamJWZGpNMEZ4U2t0WWNtcGxVRXN6TDJnMGNIa0taME00Y0Rkdk5FRkJRVWRvVm10d2JFeDNRVUZDUVUxQlVucENSa0ZwUVZsNlkyMWhjazlHU1hCRFpFMUlUSE5YVFUxamJXbFlNM1ZwUkhGVWRISlRZZ3BvYm5WblRHdzVTMkpSU1doQlNtbHRTMVUyUjNBMlRYVnhMekV5ZW13eU5XMU5hVmh2TjFsNldUbHZVR056VTJ0YWVVZHNlVXBhZDAxQmIwZERRM0ZIQ2xOTk5EbENRVTFFUVRKclFVMUhXVU5OVVVRNFRtRnpRMGhwVUVOTVFYUnZiRlpQTlM4d2NtdzFWV1pDZVcxMmFubDNhVEJPVmt0QlNtUmpRWFpRTmpFS0wwbEtiRkppV0RWSldtSmhkRWtyT0ZsUGMwTk5VVU5IYUZoT01rSlVOa05VZFdGWFF6TnhkMHhHZWtGRGVWUldOSFZwYmxGVmJraFBla1ZFWXpjMGVnb3hLM1ZIVkcxSVpsWlVVa1JKYVRJdmRFWlJkWFZPVFQwS0xTMHRMUzFGVGtRZ1EwVlNWRWxHU1VOQlZFVXRMUzB0TFFvPSJ9fX19"}], "timestampVerificationData":{"rfc3161Timestamps":[{"signedTimestamp":"MIICyjADAgEAMIICwQYJKoZIhvcNAQcCoIICsjCCAq4CAQMxDTALBglghkgBZQMEAgEwgbcGCyqGSIb3DQEJEAEEoIGnBIGkMIGhAgEBBgkrBgEEAYO/MAIwMTANBglghkgBZQMEAgEFAAQgawL8w2Yhv+alAQlEIxeoeY+ZtrXWuMVUorIiTtT4TyICFGiL4yQnFEcyiQjCaUOyUI1utRZ8GA8yMDI2MTAxOTIyMzExOFowAwIBAaAypDAwLjEVMBMGA1UEChMMc2lnc3RvcmUuZGV2MRUwEwYDVQQDEwxzaWdzdG9yZS10c2GgADGCAdwwggHYAgEBMFEwOTEVMBMGA1UEChMMc2lnc3RvcmUuZGV2MSAwHgYDVQQDExdzaWdzdG9yZS10c2Etc2VsZnNpZ25lZAIUOhNULwyQYe68wUMvy4qOiyojiwwwCwYJYIZIAWUDBAIBoIH8MBoGCSqGSIb3DQEJAzENBgsqhkiG9w0BCRABBDAcBgkqhkiG9w0BCQUxDxcNMjYxMDE5MjIzMTE4WjAvBgkqhkiG9w0BCQQxIgQgKQVaP34iASGj9tDKvbngVn5AU9ONoeeIIpUzaQv1CrQwgY4GCyqGSIb3DQEJEAIvMX8wfTB7MHkEIIX5J7wHq2LKw7RDVsEO/IGyxog/2nq55thw2dE6zQW3MFUwPaQ7MDkxFTATBgNVBAoTDHNpZ3N0b3JlLmRldjEgMB4GA1UEAxMXc2lnc3RvcmUtdHNhLXNlbGZzaWduZWQCFDoTVC8MkGHuvMFDL8uKjosqI4sMMAoGCCqGSM49BAMCBGgwZgIxAOYdsdmqMP1jGpCvbUMDAMn9a7y2Yi0TW0qE804eLILU3GezzrxsPYzwIMjo22GsAAIxAKw3FmgrW1NuQOEFdtHwvQwje7S8o7tQedDvPWScBIyBcVGsh47i6D9JjfP8hsQAyA=="}]}}, "messageSignature":{"messageDigest":{"algorithm":"SHA2_256", "digest":"AIupKj7MkgnQsFGZ7aPLG6TsiMPl+LSvrAD7IZiS20w="}, "signature":"MEQCH1pWo7lERDmxEfY7zfg2Plr2Soljeduk6BdhjaMljfACIQC1ZfK8cuETDhunkODt187M2pvTi8EVs84XL/467DkWmQ=="}}
|
build/torch213-rocm71-x86_64-linux/mha.py
ADDED
|
@@ -0,0 +1,332 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""FlashAttention forward entry points backed by aiter's Composable-Kernel FMHA.
|
| 2 |
+
|
| 3 |
+
This is the compiled (HIP / Composable Kernel) counterpart to the Triton
|
| 4 |
+
``aiter-flash-attn`` package. Only the *forward* pass is provided (dense and
|
| 5 |
+
variable-length), including support for learnable attention sinks via
|
| 6 |
+
``window_size[2]`` / ``s_aux`` (e.g. gpt-oss). The op set ships exclusively
|
| 7 |
+
the CK ``mha_fwd`` / ``mha_varlen_fwd`` kernels, so the sink path is always
|
| 8 |
+
taken (no fall-through to a non-sink ASM kernel).
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
from typing import Optional, Tuple, Union
|
| 12 |
+
|
| 13 |
+
import torch
|
| 14 |
+
from torch import Tensor
|
| 15 |
+
|
| 16 |
+
from ._ops import ops
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def _round_multiple(x: int, m: int) -> int:
|
| 20 |
+
return (x + m - 1) // m * m
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def _maybe_pad_headdim(t: Tensor, target: int) -> Tensor:
|
| 24 |
+
og = t.size(-1)
|
| 25 |
+
if og == target:
|
| 26 |
+
return t
|
| 27 |
+
return torch.nn.functional.pad(t, [0, target - og])
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def _unpack_window(window_size) -> Tuple[int, int, int]:
|
| 31 |
+
left = int(window_size[0])
|
| 32 |
+
right = int(window_size[1])
|
| 33 |
+
sink = int(window_size[2]) if len(window_size) == 3 else 0
|
| 34 |
+
return left, right, sink
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def _prep_sink(s_aux: Optional[Tensor], nheads: int, device) -> Optional[Tensor]:
|
| 38 |
+
if s_aux is None:
|
| 39 |
+
return None
|
| 40 |
+
assert s_aux.device == device, "s_aux must be on the same device as q"
|
| 41 |
+
assert s_aux.shape[0] == nheads, "s_aux must have shape [nheads_q]"
|
| 42 |
+
if s_aux.dtype != torch.float32:
|
| 43 |
+
s_aux = s_aux.to(torch.float32)
|
| 44 |
+
return s_aux
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def _effective_sink_size(sink_size: int, s_aux: Optional[Tensor]) -> int:
|
| 48 |
+
# The CK kernel only takes the sink path when sink_size > 0. Callers using
|
| 49 |
+
# the transformers attention interface pass the per-head sink logits as
|
| 50 |
+
# ``s_aux`` but leave ``window_size`` a 2-tuple (sink_size == 0), so enable
|
| 51 |
+
# a single learnable sink automatically when s_aux is given.
|
| 52 |
+
if s_aux is not None and sink_size == 0:
|
| 53 |
+
return 1
|
| 54 |
+
return sink_size
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def flash_attn_func(
|
| 58 |
+
q: Tensor,
|
| 59 |
+
k: Tensor,
|
| 60 |
+
v: Tensor,
|
| 61 |
+
dropout_p: float = 0.0,
|
| 62 |
+
softmax_scale: Optional[float] = None,
|
| 63 |
+
causal: bool = False,
|
| 64 |
+
window_size: Tuple[int, ...] = (-1, -1, 0),
|
| 65 |
+
bias: Optional[Tensor] = None,
|
| 66 |
+
alibi_slopes: Optional[Tensor] = None,
|
| 67 |
+
return_lse: bool = False,
|
| 68 |
+
return_attn_probs: bool = False,
|
| 69 |
+
s_aux: Optional[Tensor] = None,
|
| 70 |
+
**kwargs,
|
| 71 |
+
) -> Union[Tensor, Tuple[Tensor, ...]]:
|
| 72 |
+
"""Dense FlashAttention forward.
|
| 73 |
+
|
| 74 |
+
``q, k, v`` have shape ``(batch, seqlen, nheads, headdim)``. ``window_size``
|
| 75 |
+
is ``(left, right, sink_size)``; ``s_aux`` is an optional ``(nheads_q,)``
|
| 76 |
+
fp32 tensor of learnable sink logits. Extra keyword arguments (e.g. those
|
| 77 |
+
passed by the transformers flash-attention interface) are ignored.
|
| 78 |
+
"""
|
| 79 |
+
if softmax_scale is None:
|
| 80 |
+
softmax_scale = q.shape[-1] ** (-0.5)
|
| 81 |
+
window_size_left, window_size_right, sink_size = _unpack_window(window_size)
|
| 82 |
+
|
| 83 |
+
nheads_q = q.size(2)
|
| 84 |
+
s_aux = _prep_sink(s_aux, nheads_q, q.device)
|
| 85 |
+
sink_size = _effective_sink_size(sink_size, s_aux)
|
| 86 |
+
|
| 87 |
+
head_q_og = q.size(-1)
|
| 88 |
+
head_v_og = v.size(-1)
|
| 89 |
+
head_q = _round_multiple(head_q_og, 8)
|
| 90 |
+
head_v = _round_multiple(head_v_og, 8)
|
| 91 |
+
q = _maybe_pad_headdim(q, head_q)
|
| 92 |
+
k = _maybe_pad_headdim(k, head_q)
|
| 93 |
+
v = _maybe_pad_headdim(v, head_v)
|
| 94 |
+
|
| 95 |
+
return_softmax = return_attn_probs and dropout_p > 0
|
| 96 |
+
out, softmax_lse, S_dmask, _ = ops.mha_fwd(
|
| 97 |
+
q,
|
| 98 |
+
k,
|
| 99 |
+
v,
|
| 100 |
+
dropout_p,
|
| 101 |
+
softmax_scale,
|
| 102 |
+
causal,
|
| 103 |
+
window_size_left,
|
| 104 |
+
window_size_right,
|
| 105 |
+
sink_size,
|
| 106 |
+
return_lse or return_attn_probs,
|
| 107 |
+
return_softmax,
|
| 108 |
+
None, # cu_seqlens_q
|
| 109 |
+
None, # cu_seqlens_kv
|
| 110 |
+
None, # out
|
| 111 |
+
bias,
|
| 112 |
+
alibi_slopes,
|
| 113 |
+
None, # q_descale
|
| 114 |
+
None, # k_descale
|
| 115 |
+
None, # v_descale
|
| 116 |
+
s_aux,
|
| 117 |
+
None, # gen
|
| 118 |
+
)
|
| 119 |
+
|
| 120 |
+
out = out[..., :head_v_og]
|
| 121 |
+
if not (return_lse or return_attn_probs):
|
| 122 |
+
return out
|
| 123 |
+
result = (out,)
|
| 124 |
+
if return_lse:
|
| 125 |
+
result = result + (softmax_lse,)
|
| 126 |
+
if return_attn_probs:
|
| 127 |
+
result = result + (S_dmask,)
|
| 128 |
+
return result
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
def flash_attn_varlen_func(
|
| 132 |
+
q: Tensor,
|
| 133 |
+
k: Tensor,
|
| 134 |
+
v: Tensor,
|
| 135 |
+
cu_seqlens_q: Tensor,
|
| 136 |
+
cu_seqlens_k: Tensor,
|
| 137 |
+
max_seqlen_q: int,
|
| 138 |
+
max_seqlen_k: int,
|
| 139 |
+
dropout_p: float = 0.0,
|
| 140 |
+
softmax_scale: Optional[float] = None,
|
| 141 |
+
causal: bool = False,
|
| 142 |
+
window_size: Tuple[int, ...] = (-1, -1, 0),
|
| 143 |
+
bias: Optional[Tensor] = None,
|
| 144 |
+
alibi_slopes: Optional[Tensor] = None,
|
| 145 |
+
return_lse: bool = False,
|
| 146 |
+
return_attn_probs: bool = False,
|
| 147 |
+
block_table: Optional[Tensor] = None,
|
| 148 |
+
min_seqlen_q: int = 0,
|
| 149 |
+
logits_soft_cap: float = 0.0,
|
| 150 |
+
zero_tensors: bool = False,
|
| 151 |
+
cu_seqlens_q_padded: Optional[Tensor] = None,
|
| 152 |
+
cu_seqlens_k_padded: Optional[Tensor] = None,
|
| 153 |
+
s_aux: Optional[Tensor] = None,
|
| 154 |
+
**kwargs,
|
| 155 |
+
) -> Union[Tensor, Tuple[Tensor, ...]]:
|
| 156 |
+
"""Variable-length / packed FlashAttention forward.
|
| 157 |
+
|
| 158 |
+
``q`` has shape ``(total_q, nheads, headdim)``; ``cu_seqlens_*`` are int32
|
| 159 |
+
cumulative offsets of shape ``(batch + 1,)``. Sinks are supported through
|
| 160 |
+
``window_size[2]`` / ``s_aux`` as in :func:`flash_attn_func`. Extra keyword
|
| 161 |
+
arguments (e.g. from the transformers flash-attention interface) are ignored.
|
| 162 |
+
"""
|
| 163 |
+
if softmax_scale is None:
|
| 164 |
+
softmax_scale = q.shape[-1] ** (-0.5)
|
| 165 |
+
window_size_left, window_size_right, sink_size = _unpack_window(window_size)
|
| 166 |
+
|
| 167 |
+
nheads_q = q.size(1)
|
| 168 |
+
s_aux = _prep_sink(s_aux, nheads_q, q.device)
|
| 169 |
+
sink_size = _effective_sink_size(sink_size, s_aux)
|
| 170 |
+
|
| 171 |
+
head_q_og = q.size(-1)
|
| 172 |
+
head_v_og = v.size(-1)
|
| 173 |
+
head_q = _round_multiple(head_q_og, 8)
|
| 174 |
+
head_v = _round_multiple(head_v_og, 8)
|
| 175 |
+
q = _maybe_pad_headdim(q, head_q)
|
| 176 |
+
k = _maybe_pad_headdim(k, head_q)
|
| 177 |
+
v = _maybe_pad_headdim(v, head_v)
|
| 178 |
+
|
| 179 |
+
return_softmax = return_attn_probs and dropout_p > 0
|
| 180 |
+
out, softmax_lse, S_dmask, _ = ops.mha_varlen_fwd(
|
| 181 |
+
q,
|
| 182 |
+
k,
|
| 183 |
+
v,
|
| 184 |
+
cu_seqlens_q,
|
| 185 |
+
cu_seqlens_k,
|
| 186 |
+
max_seqlen_q,
|
| 187 |
+
max_seqlen_k,
|
| 188 |
+
min_seqlen_q,
|
| 189 |
+
dropout_p,
|
| 190 |
+
softmax_scale,
|
| 191 |
+
logits_soft_cap,
|
| 192 |
+
zero_tensors,
|
| 193 |
+
causal,
|
| 194 |
+
window_size_left,
|
| 195 |
+
window_size_right,
|
| 196 |
+
sink_size,
|
| 197 |
+
return_lse or return_attn_probs,
|
| 198 |
+
return_softmax,
|
| 199 |
+
None, # out
|
| 200 |
+
block_table,
|
| 201 |
+
bias,
|
| 202 |
+
alibi_slopes,
|
| 203 |
+
None, # q_descale
|
| 204 |
+
None, # k_descale
|
| 205 |
+
None, # v_descale
|
| 206 |
+
None, # gen
|
| 207 |
+
cu_seqlens_q_padded,
|
| 208 |
+
cu_seqlens_k_padded,
|
| 209 |
+
s_aux,
|
| 210 |
+
)
|
| 211 |
+
|
| 212 |
+
out = out[..., :head_v_og]
|
| 213 |
+
if not (return_lse or return_attn_probs):
|
| 214 |
+
return out
|
| 215 |
+
result = (out,)
|
| 216 |
+
if return_lse:
|
| 217 |
+
result = result + (softmax_lse,)
|
| 218 |
+
if return_attn_probs:
|
| 219 |
+
result = result + (S_dmask,)
|
| 220 |
+
return result
|
| 221 |
+
|
| 222 |
+
|
| 223 |
+
def _append_to_contiguous_cache(k_cache, v_cache, k, v, cache_seqlens, Sq):
|
| 224 |
+
"""In-place append of new k/v into a contiguous cache at per-batch offsets."""
|
| 225 |
+
B = k_cache.shape[0]
|
| 226 |
+
if isinstance(cache_seqlens, int):
|
| 227 |
+
cache_seqlens = torch.full((B,), cache_seqlens, device=k_cache.device, dtype=torch.long)
|
| 228 |
+
pos = cache_seqlens.view(B, 1).to(torch.long) + torch.arange(Sq, device=k_cache.device).view(1, Sq)
|
| 229 |
+
bidx = torch.arange(B, device=k_cache.device).view(B, 1).expand(B, Sq)
|
| 230 |
+
k_cache[bidx, pos] = k.to(k_cache.dtype)
|
| 231 |
+
v_cache[bidx, pos] = v.to(v_cache.dtype)
|
| 232 |
+
|
| 233 |
+
|
| 234 |
+
def flash_attn_with_kvcache(
|
| 235 |
+
q: Tensor,
|
| 236 |
+
k_cache: Tensor,
|
| 237 |
+
v_cache: Tensor,
|
| 238 |
+
k: Optional[Tensor] = None,
|
| 239 |
+
v: Optional[Tensor] = None,
|
| 240 |
+
rotary_cos: Optional[Tensor] = None,
|
| 241 |
+
rotary_sin: Optional[Tensor] = None,
|
| 242 |
+
cache_seqlens=None,
|
| 243 |
+
block_table: Optional[Tensor] = None,
|
| 244 |
+
softmax_scale: Optional[float] = None,
|
| 245 |
+
causal: bool = True,
|
| 246 |
+
window_size: Tuple[int, ...] = (-1, -1, 0),
|
| 247 |
+
s_aux: Optional[Tensor] = None,
|
| 248 |
+
return_lse: bool = False,
|
| 249 |
+
**kwargs,
|
| 250 |
+
) -> Union[Tensor, Tuple[Tensor, ...]]:
|
| 251 |
+
"""FlashAttention against a KV cache, built on the CK split-KV kernel.
|
| 252 |
+
|
| 253 |
+
``q`` is ``(batch, seqlen_q, nheads, headdim)``. For a contiguous cache,
|
| 254 |
+
``k_cache``/``v_cache`` are ``(batch, seqlen_cache, nheads_k, headdim)`` and
|
| 255 |
+
``cache_seqlens`` (int or ``(batch,)``) is the number of valid cache entries
|
| 256 |
+
*before* appending. If ``k``/``v`` (the new tokens) are given, they are
|
| 257 |
+
written into the cache in place. The query attends to the valid prefix
|
| 258 |
+
(``cache_seqlens`` + ``seqlen_q``) with bottom-right causal alignment.
|
| 259 |
+
|
| 260 |
+
Notes / current limitations:
|
| 261 |
+
- Rotary embedding is not applied here; pass ``q``/``k`` already rotated
|
| 262 |
+
(``rotary_cos``/``rotary_sin`` are rejected).
|
| 263 |
+
- Paged caches are addressed via ``block_table`` (forwarded to the split-KV
|
| 264 |
+
varlen kernel); in-cache append for paged layouts must be done by the
|
| 265 |
+
caller (pass ``k=v=None``).
|
| 266 |
+
"""
|
| 267 |
+
if rotary_cos is not None or rotary_sin is not None:
|
| 268 |
+
raise NotImplementedError(
|
| 269 |
+
"flash_attn_with_kvcache does not apply rotary; rotate q/k beforehand."
|
| 270 |
+
)
|
| 271 |
+
if softmax_scale is None:
|
| 272 |
+
softmax_scale = q.shape[-1] ** (-0.5)
|
| 273 |
+
window_size_left, window_size_right, sink_size = _unpack_window(window_size)
|
| 274 |
+
B, Sq, Hq, D = q.shape
|
| 275 |
+
s_aux = _prep_sink(s_aux, Hq, q.device)
|
| 276 |
+
sink_size = _effective_sink_size(sink_size, s_aux)
|
| 277 |
+
|
| 278 |
+
paged = block_table is not None
|
| 279 |
+
|
| 280 |
+
# ---- append new tokens into the cache (contiguous layout only) ----
|
| 281 |
+
if k is not None and not paged:
|
| 282 |
+
assert cache_seqlens is not None, "cache_seqlens required to append into the cache"
|
| 283 |
+
_append_to_contiguous_cache(k_cache, v_cache, k, v, cache_seqlens, Sq)
|
| 284 |
+
elif k is not None and paged:
|
| 285 |
+
raise NotImplementedError(
|
| 286 |
+
"in-cache append for paged layouts is not supported; append externally "
|
| 287 |
+
"and call with k=v=None."
|
| 288 |
+
)
|
| 289 |
+
|
| 290 |
+
appended = 0 if k is None else Sq
|
| 291 |
+
|
| 292 |
+
# ---- key lengths ----
|
| 293 |
+
if cache_seqlens is None:
|
| 294 |
+
klens = torch.full((B,), k_cache.shape[1], device=q.device, dtype=torch.long)
|
| 295 |
+
elif isinstance(cache_seqlens, int):
|
| 296 |
+
klens = torch.full((B,), cache_seqlens + appended, device=q.device, dtype=torch.long)
|
| 297 |
+
else:
|
| 298 |
+
klens = cache_seqlens.to(q.device, torch.long) + appended
|
| 299 |
+
|
| 300 |
+
uniform = bool((klens == klens[0]).all().item())
|
| 301 |
+
|
| 302 |
+
# ---- fast path: contiguous cache, uniform length -> dense kernel ----
|
| 303 |
+
if not paged and uniform:
|
| 304 |
+
L = int(klens[0].item())
|
| 305 |
+
return flash_attn_func(
|
| 306 |
+
q, k_cache[:, :L], v_cache[:, :L],
|
| 307 |
+
softmax_scale=softmax_scale, causal=causal,
|
| 308 |
+
window_size=(window_size_left, window_size_right, sink_size),
|
| 309 |
+
s_aux=s_aux, return_lse=return_lse,
|
| 310 |
+
)
|
| 311 |
+
|
| 312 |
+
# ---- general path: variable cache lengths -> split-KV varlen kernel ----
|
| 313 |
+
cu_q = torch.arange(0, (B + 1) * Sq, Sq, device=q.device, dtype=torch.int32)
|
| 314 |
+
cu_k = torch.zeros(B + 1, device=q.device, dtype=torch.int32)
|
| 315 |
+
cu_k[1:] = torch.cumsum(klens, 0).to(torch.int32)
|
| 316 |
+
q_packed = q.reshape(B * Sq, Hq, D)
|
| 317 |
+
if paged:
|
| 318 |
+
k_packed, v_packed = k_cache, v_cache # kernel gathers via block_table
|
| 319 |
+
else:
|
| 320 |
+
k_packed = torch.cat([k_cache[b, : int(klens[b])] for b in range(B)], dim=0)
|
| 321 |
+
v_packed = torch.cat([v_cache[b, : int(klens[b])] for b in range(B)], dim=0)
|
| 322 |
+
|
| 323 |
+
out = flash_attn_varlen_func(
|
| 324 |
+
q_packed, k_packed, v_packed, cu_q, cu_k, Sq, int(klens.max().item()),
|
| 325 |
+
softmax_scale=softmax_scale, causal=causal,
|
| 326 |
+
window_size=(window_size_left, window_size_right, sink_size),
|
| 327 |
+
block_table=block_table, s_aux=s_aux, return_lse=return_lse,
|
| 328 |
+
)
|
| 329 |
+
if return_lse:
|
| 330 |
+
out, lse = out
|
| 331 |
+
return out.reshape(B, Sq, Hq, D), lse
|
| 332 |
+
return out.reshape(B, Sq, Hq, D)
|
build/torch213-rocm71-x86_64-linux/symbols.json
ADDED
|
@@ -0,0 +1,349 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": 1,
|
| 3 |
+
"module": "_aiter_flash_attn_ck_rocm_fe2bb66",
|
| 4 |
+
"functions": [
|
| 5 |
+
{
|
| 6 |
+
"name": "flash_attn_func",
|
| 7 |
+
"qualname": "flash_attn_func",
|
| 8 |
+
"module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha",
|
| 9 |
+
"kind": "function",
|
| 10 |
+
"doc": "Dense FlashAttention forward.\n\n``q, k, v`` have shape ``(batch, seqlen, nheads, headdim)``. ``window_size``\nis ``(left, right, sink_size)``; ``s_aux`` is an optional ``(nheads_q,)``\nfp32 tensor of learnable sink logits. Extra keyword arguments (e.g. those\npassed by the transformers flash-attention interface) are ignored.",
|
| 11 |
+
"signature": {
|
| 12 |
+
"parameters": [
|
| 13 |
+
{
|
| 14 |
+
"name": "q",
|
| 15 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 16 |
+
"default": null,
|
| 17 |
+
"annotation": "torch.Tensor"
|
| 18 |
+
},
|
| 19 |
+
{
|
| 20 |
+
"name": "k",
|
| 21 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 22 |
+
"default": null,
|
| 23 |
+
"annotation": "torch.Tensor"
|
| 24 |
+
},
|
| 25 |
+
{
|
| 26 |
+
"name": "v",
|
| 27 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 28 |
+
"default": null,
|
| 29 |
+
"annotation": "torch.Tensor"
|
| 30 |
+
},
|
| 31 |
+
{
|
| 32 |
+
"name": "dropout_p",
|
| 33 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 34 |
+
"default": "0.0",
|
| 35 |
+
"annotation": "float"
|
| 36 |
+
},
|
| 37 |
+
{
|
| 38 |
+
"name": "softmax_scale",
|
| 39 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 40 |
+
"default": "None",
|
| 41 |
+
"annotation": "float | None"
|
| 42 |
+
},
|
| 43 |
+
{
|
| 44 |
+
"name": "causal",
|
| 45 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 46 |
+
"default": "False",
|
| 47 |
+
"annotation": "bool"
|
| 48 |
+
},
|
| 49 |
+
{
|
| 50 |
+
"name": "window_size",
|
| 51 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 52 |
+
"default": "(-1, -1, 0)",
|
| 53 |
+
"annotation": "Tuple[int, ...]"
|
| 54 |
+
},
|
| 55 |
+
{
|
| 56 |
+
"name": "bias",
|
| 57 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 58 |
+
"default": "None",
|
| 59 |
+
"annotation": "torch.Tensor | None"
|
| 60 |
+
},
|
| 61 |
+
{
|
| 62 |
+
"name": "alibi_slopes",
|
| 63 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 64 |
+
"default": "None",
|
| 65 |
+
"annotation": "torch.Tensor | None"
|
| 66 |
+
},
|
| 67 |
+
{
|
| 68 |
+
"name": "return_lse",
|
| 69 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 70 |
+
"default": "False",
|
| 71 |
+
"annotation": "bool"
|
| 72 |
+
},
|
| 73 |
+
{
|
| 74 |
+
"name": "return_attn_probs",
|
| 75 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 76 |
+
"default": "False",
|
| 77 |
+
"annotation": "bool"
|
| 78 |
+
},
|
| 79 |
+
{
|
| 80 |
+
"name": "s_aux",
|
| 81 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 82 |
+
"default": "None",
|
| 83 |
+
"annotation": "torch.Tensor | None"
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"name": "kwargs",
|
| 87 |
+
"kind": "VAR_KEYWORD",
|
| 88 |
+
"default": null,
|
| 89 |
+
"annotation": null
|
| 90 |
+
}
|
| 91 |
+
],
|
| 92 |
+
"return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]"
|
| 93 |
+
}
|
| 94 |
+
},
|
| 95 |
+
{
|
| 96 |
+
"name": "flash_attn_varlen_func",
|
| 97 |
+
"qualname": "flash_attn_varlen_func",
|
| 98 |
+
"module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha",
|
| 99 |
+
"kind": "function",
|
| 100 |
+
"doc": "Variable-length / packed FlashAttention forward.\n\n``q`` has shape ``(total_q, nheads, headdim)``; ``cu_seqlens_*`` are int32\ncumulative offsets of shape ``(batch + 1,)``. Sinks are supported through\n``window_size[2]`` / ``s_aux`` as in :func:`flash_attn_func`. Extra keyword\narguments (e.g. from the transformers flash-attention interface) are ignored.",
|
| 101 |
+
"signature": {
|
| 102 |
+
"parameters": [
|
| 103 |
+
{
|
| 104 |
+
"name": "q",
|
| 105 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 106 |
+
"default": null,
|
| 107 |
+
"annotation": "torch.Tensor"
|
| 108 |
+
},
|
| 109 |
+
{
|
| 110 |
+
"name": "k",
|
| 111 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 112 |
+
"default": null,
|
| 113 |
+
"annotation": "torch.Tensor"
|
| 114 |
+
},
|
| 115 |
+
{
|
| 116 |
+
"name": "v",
|
| 117 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 118 |
+
"default": null,
|
| 119 |
+
"annotation": "torch.Tensor"
|
| 120 |
+
},
|
| 121 |
+
{
|
| 122 |
+
"name": "cu_seqlens_q",
|
| 123 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 124 |
+
"default": null,
|
| 125 |
+
"annotation": "torch.Tensor"
|
| 126 |
+
},
|
| 127 |
+
{
|
| 128 |
+
"name": "cu_seqlens_k",
|
| 129 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 130 |
+
"default": null,
|
| 131 |
+
"annotation": "torch.Tensor"
|
| 132 |
+
},
|
| 133 |
+
{
|
| 134 |
+
"name": "max_seqlen_q",
|
| 135 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 136 |
+
"default": null,
|
| 137 |
+
"annotation": "int"
|
| 138 |
+
},
|
| 139 |
+
{
|
| 140 |
+
"name": "max_seqlen_k",
|
| 141 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 142 |
+
"default": null,
|
| 143 |
+
"annotation": "int"
|
| 144 |
+
},
|
| 145 |
+
{
|
| 146 |
+
"name": "dropout_p",
|
| 147 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 148 |
+
"default": "0.0",
|
| 149 |
+
"annotation": "float"
|
| 150 |
+
},
|
| 151 |
+
{
|
| 152 |
+
"name": "softmax_scale",
|
| 153 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 154 |
+
"default": "None",
|
| 155 |
+
"annotation": "float | None"
|
| 156 |
+
},
|
| 157 |
+
{
|
| 158 |
+
"name": "causal",
|
| 159 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 160 |
+
"default": "False",
|
| 161 |
+
"annotation": "bool"
|
| 162 |
+
},
|
| 163 |
+
{
|
| 164 |
+
"name": "window_size",
|
| 165 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 166 |
+
"default": "(-1, -1, 0)",
|
| 167 |
+
"annotation": "Tuple[int, ...]"
|
| 168 |
+
},
|
| 169 |
+
{
|
| 170 |
+
"name": "bias",
|
| 171 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 172 |
+
"default": "None",
|
| 173 |
+
"annotation": "torch.Tensor | None"
|
| 174 |
+
},
|
| 175 |
+
{
|
| 176 |
+
"name": "alibi_slopes",
|
| 177 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 178 |
+
"default": "None",
|
| 179 |
+
"annotation": "torch.Tensor | None"
|
| 180 |
+
},
|
| 181 |
+
{
|
| 182 |
+
"name": "return_lse",
|
| 183 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 184 |
+
"default": "False",
|
| 185 |
+
"annotation": "bool"
|
| 186 |
+
},
|
| 187 |
+
{
|
| 188 |
+
"name": "return_attn_probs",
|
| 189 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 190 |
+
"default": "False",
|
| 191 |
+
"annotation": "bool"
|
| 192 |
+
},
|
| 193 |
+
{
|
| 194 |
+
"name": "block_table",
|
| 195 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 196 |
+
"default": "None",
|
| 197 |
+
"annotation": "torch.Tensor | None"
|
| 198 |
+
},
|
| 199 |
+
{
|
| 200 |
+
"name": "min_seqlen_q",
|
| 201 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 202 |
+
"default": "0",
|
| 203 |
+
"annotation": "int"
|
| 204 |
+
},
|
| 205 |
+
{
|
| 206 |
+
"name": "logits_soft_cap",
|
| 207 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 208 |
+
"default": "0.0",
|
| 209 |
+
"annotation": "float"
|
| 210 |
+
},
|
| 211 |
+
{
|
| 212 |
+
"name": "zero_tensors",
|
| 213 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 214 |
+
"default": "False",
|
| 215 |
+
"annotation": "bool"
|
| 216 |
+
},
|
| 217 |
+
{
|
| 218 |
+
"name": "cu_seqlens_q_padded",
|
| 219 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 220 |
+
"default": "None",
|
| 221 |
+
"annotation": "torch.Tensor | None"
|
| 222 |
+
},
|
| 223 |
+
{
|
| 224 |
+
"name": "cu_seqlens_k_padded",
|
| 225 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 226 |
+
"default": "None",
|
| 227 |
+
"annotation": "torch.Tensor | None"
|
| 228 |
+
},
|
| 229 |
+
{
|
| 230 |
+
"name": "s_aux",
|
| 231 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 232 |
+
"default": "None",
|
| 233 |
+
"annotation": "torch.Tensor | None"
|
| 234 |
+
},
|
| 235 |
+
{
|
| 236 |
+
"name": "kwargs",
|
| 237 |
+
"kind": "VAR_KEYWORD",
|
| 238 |
+
"default": null,
|
| 239 |
+
"annotation": null
|
| 240 |
+
}
|
| 241 |
+
],
|
| 242 |
+
"return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]"
|
| 243 |
+
}
|
| 244 |
+
},
|
| 245 |
+
{
|
| 246 |
+
"name": "flash_attn_with_kvcache",
|
| 247 |
+
"qualname": "flash_attn_with_kvcache",
|
| 248 |
+
"module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha",
|
| 249 |
+
"kind": "function",
|
| 250 |
+
"doc": "FlashAttention against a KV cache, built on the CK split-KV kernel.\n\n``q`` is ``(batch, seqlen_q, nheads, headdim)``. For a contiguous cache,\n``k_cache``/``v_cache`` are ``(batch, seqlen_cache, nheads_k, headdim)`` and\n``cache_seqlens`` (int or ``(batch,)``) is the number of valid cache entries\n*before* appending. If ``k``/``v`` (the new tokens) are given, they are\nwritten into the cache in place. The query attends to the valid prefix\n(``cache_seqlens`` + ``seqlen_q``) with bottom-right causal alignment.\n\nNotes / current limitations:\n- Rotary embedding is not applied here; pass ``q``/``k`` already rotated\n (``rotary_cos``/``rotary_sin`` are rejected).\n- Paged caches are addressed via ``block_table`` (forwarded to the split-KV\n varlen kernel); in-cache append for paged layouts must be done by the\n caller (pass ``k=v=None``).",
|
| 251 |
+
"signature": {
|
| 252 |
+
"parameters": [
|
| 253 |
+
{
|
| 254 |
+
"name": "q",
|
| 255 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 256 |
+
"default": null,
|
| 257 |
+
"annotation": "torch.Tensor"
|
| 258 |
+
},
|
| 259 |
+
{
|
| 260 |
+
"name": "k_cache",
|
| 261 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 262 |
+
"default": null,
|
| 263 |
+
"annotation": "torch.Tensor"
|
| 264 |
+
},
|
| 265 |
+
{
|
| 266 |
+
"name": "v_cache",
|
| 267 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 268 |
+
"default": null,
|
| 269 |
+
"annotation": "torch.Tensor"
|
| 270 |
+
},
|
| 271 |
+
{
|
| 272 |
+
"name": "k",
|
| 273 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 274 |
+
"default": "None",
|
| 275 |
+
"annotation": "torch.Tensor | None"
|
| 276 |
+
},
|
| 277 |
+
{
|
| 278 |
+
"name": "v",
|
| 279 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 280 |
+
"default": "None",
|
| 281 |
+
"annotation": "torch.Tensor | None"
|
| 282 |
+
},
|
| 283 |
+
{
|
| 284 |
+
"name": "rotary_cos",
|
| 285 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 286 |
+
"default": "None",
|
| 287 |
+
"annotation": "torch.Tensor | None"
|
| 288 |
+
},
|
| 289 |
+
{
|
| 290 |
+
"name": "rotary_sin",
|
| 291 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 292 |
+
"default": "None",
|
| 293 |
+
"annotation": "torch.Tensor | None"
|
| 294 |
+
},
|
| 295 |
+
{
|
| 296 |
+
"name": "cache_seqlens",
|
| 297 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 298 |
+
"default": "None",
|
| 299 |
+
"annotation": null
|
| 300 |
+
},
|
| 301 |
+
{
|
| 302 |
+
"name": "block_table",
|
| 303 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 304 |
+
"default": "None",
|
| 305 |
+
"annotation": "torch.Tensor | None"
|
| 306 |
+
},
|
| 307 |
+
{
|
| 308 |
+
"name": "softmax_scale",
|
| 309 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 310 |
+
"default": "None",
|
| 311 |
+
"annotation": "float | None"
|
| 312 |
+
},
|
| 313 |
+
{
|
| 314 |
+
"name": "causal",
|
| 315 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 316 |
+
"default": "True",
|
| 317 |
+
"annotation": "bool"
|
| 318 |
+
},
|
| 319 |
+
{
|
| 320 |
+
"name": "window_size",
|
| 321 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 322 |
+
"default": "(-1, -1, 0)",
|
| 323 |
+
"annotation": "Tuple[int, ...]"
|
| 324 |
+
},
|
| 325 |
+
{
|
| 326 |
+
"name": "s_aux",
|
| 327 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 328 |
+
"default": "None",
|
| 329 |
+
"annotation": "torch.Tensor | None"
|
| 330 |
+
},
|
| 331 |
+
{
|
| 332 |
+
"name": "return_lse",
|
| 333 |
+
"kind": "POSITIONAL_OR_KEYWORD",
|
| 334 |
+
"default": "False",
|
| 335 |
+
"annotation": "bool"
|
| 336 |
+
},
|
| 337 |
+
{
|
| 338 |
+
"name": "kwargs",
|
| 339 |
+
"kind": "VAR_KEYWORD",
|
| 340 |
+
"default": null,
|
| 341 |
+
"annotation": null
|
| 342 |
+
}
|
| 343 |
+
],
|
| 344 |
+
"return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]"
|
| 345 |
+
}
|
| 346 |
+
}
|
| 347 |
+
],
|
| 348 |
+
"layers": []
|
| 349 |
+
}
|
build/torch213-rocm72-x86_64-linux/__init__.py
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""AITER Composable-Kernel FlashAttention kernels for AMD ROCm (compiled HIP).
|
| 2 |
+
|
| 3 |
+
Compiled (Composable Kernel / HIP) counterpart to the Triton-based
|
| 4 |
+
``aiter-flash-attn`` package, repackaged from the CK FMHA forward path of the
|
| 5 |
+
`ROCm/aiter <https://github.com/ROCm/aiter>`_ project as a self-contained
|
| 6 |
+
Hugging Face Hub kernel. Exposes the FlashAttention forward entry points with
|
| 7 |
+
learnable attention sink support (e.g. gpt-oss).
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
from .mha import (
|
| 11 |
+
flash_attn_func,
|
| 12 |
+
flash_attn_varlen_func,
|
| 13 |
+
flash_attn_with_kvcache,
|
| 14 |
+
)
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
__kernel_metadata__ = {
|
| 18 |
+
"license": "mit",
|
| 19 |
+
}
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
__all__ = [
|
| 23 |
+
"__kernel_metadata__",
|
| 24 |
+
"flash_attn_func",
|
| 25 |
+
"flash_attn_varlen_func",
|
| 26 |
+
"flash_attn_with_kvcache",
|
| 27 |
+
]
|