kernels-bot commited on
Commit
a9d7d0f
·
verified ·
1 Parent(s): 218d14b

Uploaded using `kernel-builder`.

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. build/torch210-rocm70-x86_64-linux/__init__.py +27 -0
  2. build/torch210-rocm70-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so +3 -0
  3. build/torch210-rocm70-x86_64-linux/_ops.py +9 -0
  4. build/torch210-rocm70-x86_64-linux/metadata.json +37 -0
  5. build/torch210-rocm70-x86_64-linux/metadata.json.sigstore +1 -0
  6. build/torch210-rocm70-x86_64-linux/mha.py +332 -0
  7. build/torch210-rocm70-x86_64-linux/symbols.json +349 -0
  8. build/torch210-rocm71-x86_64-linux/__init__.py +27 -0
  9. build/torch210-rocm71-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so +3 -0
  10. build/torch210-rocm71-x86_64-linux/_ops.py +9 -0
  11. build/torch210-rocm71-x86_64-linux/metadata.json +37 -0
  12. build/torch210-rocm71-x86_64-linux/metadata.json.sigstore +1 -0
  13. build/torch210-rocm71-x86_64-linux/mha.py +332 -0
  14. build/torch210-rocm71-x86_64-linux/symbols.json +349 -0
  15. build/torch211-rocm71-x86_64-linux/__init__.py +27 -0
  16. build/torch211-rocm71-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so +3 -0
  17. build/torch211-rocm71-x86_64-linux/_ops.py +9 -0
  18. build/torch211-rocm71-x86_64-linux/metadata.json +37 -0
  19. build/torch211-rocm71-x86_64-linux/metadata.json.sigstore +1 -0
  20. build/torch211-rocm71-x86_64-linux/mha.py +332 -0
  21. build/torch211-rocm71-x86_64-linux/symbols.json +349 -0
  22. build/torch211-rocm72-x86_64-linux/__init__.py +27 -0
  23. build/torch211-rocm72-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so +3 -0
  24. build/torch211-rocm72-x86_64-linux/_ops.py +9 -0
  25. build/torch211-rocm72-x86_64-linux/metadata.json +37 -0
  26. build/torch211-rocm72-x86_64-linux/metadata.json.sigstore +1 -0
  27. build/torch211-rocm72-x86_64-linux/mha.py +332 -0
  28. build/torch211-rocm72-x86_64-linux/symbols.json +349 -0
  29. build/torch212-rocm71-x86_64-linux/__init__.py +27 -0
  30. build/torch212-rocm71-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so +3 -0
  31. build/torch212-rocm71-x86_64-linux/_ops.py +9 -0
  32. build/torch212-rocm71-x86_64-linux/metadata.json +37 -0
  33. build/torch212-rocm71-x86_64-linux/metadata.json.sigstore +1 -0
  34. build/torch212-rocm71-x86_64-linux/mha.py +332 -0
  35. build/torch212-rocm71-x86_64-linux/symbols.json +349 -0
  36. build/torch212-rocm72-x86_64-linux/__init__.py +27 -0
  37. build/torch212-rocm72-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so +3 -0
  38. build/torch212-rocm72-x86_64-linux/_ops.py +9 -0
  39. build/torch212-rocm72-x86_64-linux/metadata.json +37 -0
  40. build/torch212-rocm72-x86_64-linux/metadata.json.sigstore +1 -0
  41. build/torch212-rocm72-x86_64-linux/mha.py +332 -0
  42. build/torch212-rocm72-x86_64-linux/symbols.json +349 -0
  43. build/torch213-rocm71-x86_64-linux/__init__.py +27 -0
  44. build/torch213-rocm71-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so +3 -0
  45. build/torch213-rocm71-x86_64-linux/_ops.py +9 -0
  46. build/torch213-rocm71-x86_64-linux/metadata.json +37 -0
  47. build/torch213-rocm71-x86_64-linux/metadata.json.sigstore +1 -0
  48. build/torch213-rocm71-x86_64-linux/mha.py +332 -0
  49. build/torch213-rocm71-x86_64-linux/symbols.json +349 -0
  50. build/torch213-rocm72-x86_64-linux/__init__.py +27 -0
build/torch210-rocm70-x86_64-linux/__init__.py ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """AITER Composable-Kernel FlashAttention kernels for AMD ROCm (compiled HIP).
2
+
3
+ Compiled (Composable Kernel / HIP) counterpart to the Triton-based
4
+ ``aiter-flash-attn`` package, repackaged from the CK FMHA forward path of the
5
+ `ROCm/aiter <https://github.com/ROCm/aiter>`_ project as a self-contained
6
+ Hugging Face Hub kernel. Exposes the FlashAttention forward entry points with
7
+ learnable attention sink support (e.g. gpt-oss).
8
+ """
9
+
10
+ from .mha import (
11
+ flash_attn_func,
12
+ flash_attn_varlen_func,
13
+ flash_attn_with_kvcache,
14
+ )
15
+
16
+
17
+ __kernel_metadata__ = {
18
+ "license": "mit",
19
+ }
20
+
21
+
22
+ __all__ = [
23
+ "__kernel_metadata__",
24
+ "flash_attn_func",
25
+ "flash_attn_varlen_func",
26
+ "flash_attn_with_kvcache",
27
+ ]
build/torch210-rocm70-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8edc92287fd49261db7c4799b29691a77e7db509ec44dac8bf85fba127f0e641
3
+ size 204329576
build/torch210-rocm70-x86_64-linux/_ops.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _aiter_flash_attn_ck_rocm_fe2bb66
3
+ ops = torch.ops._aiter_flash_attn_ck_rocm_fe2bb66
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ """
7
+ Prefix op by namespace.
8
+ """
9
+ return f"_aiter_flash_attn_ck_rocm_fe2bb66::{op_name}"
build/torch210-rocm70-x86_64-linux/metadata.json ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "aiter-flash-attn-ck",
3
+ "id": "_aiter_flash_attn_ck_rocm_fe2bb66",
4
+ "version": 1,
5
+ "kernels-minver": "0.14.0",
6
+ "license": "MIT",
7
+ "python-depends": [],
8
+ "kernel-depends": [],
9
+ "backend": {
10
+ "type": "rocm",
11
+ "archs": [
12
+ "gfx942",
13
+ "gfx950"
14
+ ]
15
+ },
16
+ "digest": {
17
+ "algorithm": "sha256",
18
+ "files": {
19
+ "__init__.py": "7wj1GTm2bqTLLCyT4xLuAk/+JWYrhINjY/ODMNWhgZ0=",
20
+ "_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so": "jtySKH/UkmHbfEeZspaRp359tQnsRNrIv4X7oSfw5kE=",
21
+ "_ops.py": "4WkdYDWWJp60zep+oW/5ym9vIeVMNfrgRCkP3o/w7yU=",
22
+ "mha.py": "zCAjY4O+DiCzmEL+W9sqXxZaqjsj4RJbWFcyPpUZejk=",
23
+ "symbols.json": "rhnF4H8Jsb6I+YPxIVFMgnLcTQJODEXbZ7ge77NZXAU="
24
+ }
25
+ },
26
+ "provenance": {
27
+ "kernel-builder": {
28
+ "version": "0.18.0-dev0",
29
+ "commit": "d054906a13710b811b96697b4635f03020842733",
30
+ "dirty": false
31
+ },
32
+ "kernel": {
33
+ "commit": "fe2bb66fee6c2a77eccf08a0508dc57428d488ed",
34
+ "dirty": false
35
+ }
36
+ }
37
+ }
build/torch210-rocm70-x86_64-linux/metadata.json.sigstore ADDED
@@ -0,0 +1 @@
 
 
1
+ {"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json", "verificationMaterial":{"certificate":{"rawBytes":"MIIHLDCCBrOgAwIBAgIURaYLBvCxo8VXbDXCmafFy5NEz4IwCgYIKoZIzj0EAwMwNzEVMBMGA1UEChMMc2lnc3RvcmUuZGV2MR4wHAYDVQQDExVzaWdzdG9yZS1pbnRlcm1lZGlhdGUwHhcNMjYxMDE5MjIzMTEwWhcNMjYxMDE5MjI0MTEwWjAAMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEj/NhfZLBdSTQaFcS2gIM0IgWG5P9kCv+yNljBxYjIPdh5dPlT5SG3L0TOIhLASBQ2EJ2qqDHsaMED9/WJmH9hqOCBdIwggXOMA4GA1UdDwEB/wQEAwIHgDATBgNVHSUEDDAKBggrBgEFBQcDAzAfBgNVHSMEGDAWgBTf0+nPViQRlvmo2OkoVaLGLhhkPzBrBgNVHREBAf8EYTBfhl1odHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkvLmdpdGh1Yi93b3JrZmxvd3MvYnVpbGQueWFtbEByZWZzL2hlYWRzL21haW4wOQYKKwYBBAGDvzABAQQraHR0cHM6Ly90b2tlbi5hY3Rpb25zLmdpdGh1YnVzZXJjb250ZW50LmNvbTAfBgorBgEEAYO/MAECBBF3b3JrZmxvd19kaXNwYXRjaDA2BgorBgEEAYO/MAEDBChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMBMGCisGAQQBg78wAQQEBUJ1aWxkMCsGCisGAQQBg78wAQUEHWh1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5MB0GCisGAQQBg78wAQYED3JlZnMvaGVhZHMvbWFpbjA7BgorBgEEAYO/MAEIBC0MK2h0dHBzOi8vdG9rZW4uYWN0aW9ucy5naXRodWJ1c2VyY29udGVudC5jb20wbQYKKwYBBAGDvzABCQRfDF1odHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkvLmdpdGh1Yi93b3JrZmxvd3MvYnVpbGQueWFtbEByZWZzL2hlYWRzL21haW4wOAYKKwYBBAGDvzABCgQqDChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMBsGCisGAQQBg78wAQsEDQwLc2VsZi1ob3N0ZWQwQAYKKwYBBAGDvzABDAQyDDBodHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkwOAYKKwYBBAGDvzABDQQqDChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMB8GCisGAQQBg78wAQ4EEQwPcmVmcy9oZWFkcy9tYWluMBoGCisGAQQBg78wAQ8EDAwKMTA3MTQ3NTUyOTAuBgorBgEEAYO/MAEQBCAMHmh0dHBzOi8vZ2l0aHViLmNvbS9odWdnaW5nZmFjZTAYBgorBgEEAYO/MAERBAoMCDI1NzIwNzQzMG0GCisGAQQBg78wARIEXwxdaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5Ly5naXRodWIvd29ya2Zsb3dzL2J1aWxkLnlhbWxAcmVmcy9oZWFkcy9tYWluMDgGCisGAQQBg78wARMEKgwoZmUyYmI2NmZlZTZjMmE3N2VjY2YwOGEwNTA4ZGM1NzQyOGQ0ODhlZDAhBgorBgEEAYO/MAEUBBMMEXdvcmtmbG93X2Rpc3BhdGNoMGQGCisGAQQBg78wARUEVgxUaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5L2FjdGlvbnMvcnVucy8zOTIzOTUxODEwNC9hdHRlbXB0cy8xMBYGCisGAQQBg78wARYECAwGcHVibGljMEYGCisGAQQBg78wARgEOAw2cmVwbzpodWdnaW5nZmFjZS9rZXJuZWxzLWNvbW11bml0eTpyZWY6cmVmcy9oZWFkcy9tYWluMIGLBgorBgEEAdZ5AgQCBH0EewB5AHcA3T0wasbHETJjGR4cmWc3AqJKXrjePK3/h4pygC8p7o4AAAGhVkpFsgAABAMASDBGAiEAs9cSP0P/pRZ4prabNNIq89TS0RhX2f3JRpHryeIhR3YCIQDnQuuXxSnnnfl8Q7URBS7DC23YrDaSpshSRUENXthSezAKBggqhkjOPQQDAwNnADBkAjEA4gCCe/00DKR/78FksRs11BgMWSSsisk8ME6OMyx2ZOKIvMVy1VHI15E6Xfr3isA+Ai9ZI6Otv6AnOVw/zUHKLvuj6VoYdLp26TwhWYN2N8LFZjzVGNX+0NcN8O6rYcPUsw=="}, "tlogEntries":[{"logIndex":"3327211074", "logId":{"keyId":"wNI9atQGlz+VWfO6LRygH4QUfY/8W4RFwiT5i5WRgB0="}, "kindVersion":{"kind":"hashedrekord", "version":"0.0.1"}, "integratedTime":"1792449070", "inclusionPromise":{"signedEntryTimestamp":"MEUCIBvKLoqT8cbWQvPal/N1/qcIkVmZMOCOjE6UEkoLG/MqAiEAviJ70KCt6sreW85HkoeGGZY7csZL/nOGK/uLJB3Mo5Y="}, "inclusionProof":{"logIndex":"3205306812", "rootHash":"95EwDSNtaCqCTTYN6UQnUbg7LfPk37Dr84DcE4k1RBM=", "treeSize":"3205306841", "hashes":["Ocn28evxoK5pHLcp+On0tDY2wedQVBaFskplXSixJIA=", "mwzUJi5lok6Hybu0CxcVdWew/1Qbhdd1iGjTjEYBcZY=", "5H4knr1LMewHY/JuNzQYKyCx9Q74EBBYz3NqEsl4bQo=", "QZRIvZ2WmlY+tW/q10eYuY+Pr8vI5My3TuMAw/2EObw=", "k+ztsFzU6f6Y66AdU9NwlAXHqLCPJ6LKyl669zO1Hr4=", "ozBdZyx9tTLcaQf2DOlQzhq1u90wECTuyt20Ufxet/w=", "ddB/VAJce0tlDaSu+1n3gD4CW0cuWQCIYTwHV3ehNRg=", "mnhxYPYaPxA2/IDhvMQ+URxfzi+3aVt0cMthNxkCByA=", "jhDPXc61HrM2wJE+UCBVCldN7S5Fn1BuEfb57UT4fLc=", "5egUlZ4Pi0Ix/qLyCDXocsWywJCHqrde/S06vOZvlOM=", "QsXufUpoKtSjKAB19K1G00umQpWvPHlBNuRGFiiVjQQ=", "QIMkwQbwAewhLJd//5FFwI3SnyNQBGjCgJrelfG8vhc=", "Et17CACUv9QN64irUJuBjsrgRGkGnnJ9xacRV/aBdwQ=", "lqgEwkxhDMs/gdIAE4lmjfMwg7nDmv7lda6O0LGliCw=", "HovTPBWzoErvUg1u5sIi3x1ZonZ7yJekVsBrmpomCIA=", "G+nk7mgE5h6dUmv2UY7NDJrY3WvrZY31raaO1AETt+o=", "qlTWNDzERnTBQVcHHXXVPjiUP0SsikGxfcxVlgqVFDo=", "gg9GNApNke3+zNhl4tsWtmIycCFrOMCB3lys6XuSQNQ=", "NlOclldmKPA1tMKIskqHAwuQDYHp31YwpK+OOMrMccc=", "qxzHanAzz57SDdmJe0B7bJK72NTIBbwEMGBBKvDOROw=", "xH/DCseLHr9eKoYT8qsORZK7zVdEGYWHuVtsVrD95wY="], "checkpoint":{"envelope":"rekor.sigstore.dev - 1193050959916656506\n3205306841\n95EwDSNtaCqCTTYN6UQnUbg7LfPk37Dr84DcE4k1RBM=\n\n— rekor.sigstore.dev wNI9ajBFAiEAr3pHRHsXHEeGuhek+3BjAqkphYe6M+vo6+TlCdjCZNYCIE11bmNslf4u1xUYgIQHRJqsJ7XyHOBWS7628PjJh16z\n"}}, "canonicalizedBody":"eyJhcGlWZXJzaW9uIjoiMC4wLjEiLCJraW5kIjoiaGFzaGVkcmVrb3JkIiwic3BlYyI6eyJkYXRhIjp7Imhhc2giOnsiYWxnb3JpdGhtIjoic2hhMjU2IiwidmFsdWUiOiIwOGZhMGNkNTBhODRmOTQxMjc2YzVlNmQ1ZmE0OWYxZDRlMDFlMTU5ODdhNjBhYWI4ZjJhYjZlMzE5ZWVjNzdiIn19LCJzaWduYXR1cmUiOnsiY29udGVudCI6Ik1FVUNJQ1BTaWFYOXBZL29aYnp1YXNHQW5PUTB1TDhQL0pMSEsrRkttVUhZZnZUMkFpRUFzOEZDOEhSeVZFZmo3QUdqYnNJNlZmb2FoY2NLWHM0RkNmY3luZ1RzUVhjPSIsInB1YmxpY0tleSI6eyJjb250ZW50IjoiTFMwdExTMUNSVWRKVGlCRFJWSlVTVVpKUTBGVVJTMHRMUzB0Q2sxSlNVaE1SRU5EUW5KUFowRjNTVUpCWjBsVlVtRlpURUoyUTNodk9GWllZa1JZUTIxaFprWjVOVTVGZWpSSmQwTm5XVWxMYjFwSmVtb3dSVUYzVFhjS1RucEZWazFDVFVkQk1WVkZRMmhOVFdNeWJHNWpNMUoyWTIxVmRWcEhWakpOVWpSM1NFRlpSRlpSVVVSRmVGWjZZVmRrZW1SSE9YbGFVekZ3WW01U2JBcGpiVEZzV2tkc2FHUkhWWGRJYUdOT1RXcFplRTFFUlRWTmFrbDZUVlJGZDFkb1kwNU5hbGw0VFVSRk5VMXFTVEJOVkVWM1YycEJRVTFHYTNkRmQxbElDa3R2V2tsNmFqQkRRVkZaU1V0dldrbDZhakJFUVZGalJGRm5RVVZxTDA1b1pscE1RbVJUVkZGaFJtTlRNbWRKVFRCSloxZEhOVkE1YTBOMkszbE9iR29LUW5oWmFrbFFaR2cxWkZCc1ZEVlRSek5NTUZSUFNXaE1RVk5DVVRKRlNqSnhjVVJJYzJGTlJVUTVMMWRLYlVnNWFIRlBRMEprU1hkbloxaFBUVUUwUndwQk1WVmtSSGRGUWk5M1VVVkJkMGxJWjBSQlZFSm5UbFpJVTFWRlJFUkJTMEpuWjNKQ1owVkdRbEZqUkVGNlFXWkNaMDVXU0ZOTlJVZEVRVmRuUWxSbUNqQXJibEJXYVZGU2JIWnRiekpQYTI5V1lVeEhUR2hvYTFCNlFuSkNaMDVXU0ZKRlFrRm1PRVZaVkVKbWFHd3hiMlJJVW5kamVtOTJUREprY0dSSGFERUtXV2sxYW1JeU1IWmhTRlp1V2pKc2RWb3lXbWhaTWxWMllUSldlV0p0Vm5OamVURnFZakl4ZEdSWE5YQmtTR3QyVEcxa2NHUkhhREZaYVRrellqTktjZ3BhYlhoMlpETk5kbGx1Vm5CaVIxRjFaVmRHZEdKRlFubGFWMXA2VERKb2JGbFhVbnBNTWpGb1lWYzBkMDlSV1V0TGQxbENRa0ZIUkhaNlFVSkJVVkZ5Q21GSVVqQmpTRTAyVEhrNU1HSXlkR3hpYVRWb1dUTlNjR0l5TlhwTWJXUndaRWRvTVZsdVZucGFXRXBxWWpJMU1GcFhOVEJNYlU1MllsUkJaa0puYjNJS1FtZEZSVUZaVHk5TlFVVkRRa0pHTTJJelNuSmFiWGgyWkRFNWEyRllUbmRaV0ZKcVlVUkJNa0puYjNKQ1owVkZRVmxQTDAxQlJVUkNRMmh0V2xSS2FRcFphbGt5V20xV2JFNXRUWGxaVkdNeldsZE9hbHBxUVRSWlZFRXhUVVJvYTFsNlZUTk9SRWswV2tSUk5FOUhWbXROUWsxSFEybHpSMEZSVVVKbk56aDNDa0ZSVVVWQ1ZVb3hZVmQ0YTAxRGMwZERhWE5IUVZGUlFtYzNPSGRCVVZWRlNGZG9NVm95WkhCaWJXUnRXVmRPYkV3eWRHeGpiVFZzWWtoTmRGa3lPWFFLWWxoV2RXRllValZOUWpCSFEybHpSMEZSVVVKbk56aDNRVkZaUlVRelNteGFiazEyWVVkV2FGcElUWFppVjBad1ltcEJOMEpuYjNKQ1owVkZRVmxQTHdwTlFVVkpRa013VFVzeWFEQmtTRUo2VDJrNGRtUkhPWEphVnpSMVdWZE9NR0ZYT1hWamVUVnVZVmhTYjJSWFNqRmpNbFo1V1RJNWRXUkhWblZrUXpWcUNtSXlNSGRpVVZsTFMzZFpRa0pCUjBSMmVrRkNRMUZTWmtSR01XOWtTRkozWTNwdmRrd3laSEJrUjJneFdXazFhbUl5TUhaaFNGWnVXakpzZFZveVdtZ0tXVEpWZG1FeVZubGliVlp6WTNreGFtSXlNWFJrVnpWd1pFaHJka3h0WkhCa1IyZ3hXV2s1TTJJelNuSmFiWGgyWkROTmRsbHVWbkJpUjFGMVpWZEdkQXBpUlVKNVdsZGFla3d5YUd4WlYxSjZUREl4YUdGWE5IZFBRVmxMUzNkWlFrSkJSMFIyZWtGQ1EyZFJjVVJEYUcxYVZFcHBXV3BaTWxwdFZteE9iVTE1Q2xsVVl6TmFWMDVxV21wQk5GbFVRVEZOUkdocldYcFZNMDVFU1RSYVJGRTBUMGRXYTAxQ2MwZERhWE5IUVZGUlFtYzNPSGRCVVhORlJGRjNUR015Vm5NS1dta3hiMkl6VGpCYVYxRjNVVUZaUzB0M1dVSkNRVWRFZG5wQlFrUkJVWGxFUkVKdlpFaFNkMk42YjNaTU1tUndaRWRvTVZscE5XcGlNakIyWVVoV2JncGFNbXgxV2pKYWFGa3lWWFpoTWxaNVltMVdjMk41TVdwaU1qRjBaRmMxY0dSSWEzZFBRVmxMUzNkWlFrSkJSMFIyZWtGQ1JGRlJjVVJEYUcxYVZFcHBDbGxxV1RKYWJWWnNUbTFOZVZsVVl6TmFWMDVxV21wQk5GbFVRVEZOUkdocldYcFZNMDVFU1RSYVJGRTBUMGRXYTAxQ09FZERhWE5IUVZGUlFtYzNPSGNLUVZFMFJVVlJkMUJqYlZadFkzazViMXBYUm10amVUbDBXVmRzZFUxQ2IwZERhWE5IUVZGUlFtYzNPSGRCVVRoRlJFRjNTMDFVUVROTlZGRXpUbFJWZVFwUFZFRjFRbWR2Y2tKblJVVkJXVTh2VFVGRlVVSkRRVTFJYldnd1pFaENlazlwT0haYU1td3dZVWhXYVV4dFRuWmlVemx2WkZka2JtRlhOVzVhYlVacUNscFVRVmxDWjI5eVFtZEZSVUZaVHk5TlFVVlNRa0Z2VFVORVNURk9la2wzVG5wUmVrMUhNRWREYVhOSFFWRlJRbWMzT0hkQlVrbEZXSGQ0WkdGSVVqQUtZMGhOTmt4NU9XNWhXRkp2WkZkSmRWa3lPWFJNTW1neFdqSmtjR0p0WkcxWlYwNXNUREowYkdOdE5XeGlTRTEwV1RJNWRHSllWblZoV0ZJMVRIazFiZ3BoV0ZKdlpGZEpkbVF5T1hsaE1scHpZak5rZWt3eVNqRmhWM2hyVEc1c2FHSlhlRUZqYlZadFkzazViMXBYUm10amVUbDBXVmRzZFUxRVowZERhWE5IQ2tGUlVVSm5OemgzUVZKTlJVdG5kMjlhYlZWNVdXMUpNazV0V214YVZGcHFUVzFGTTA0eVZtcFpNbGwzVDBkRmQwNVVRVFJhUjAweFRucFJlVTlIVVRBS1QwUm9iRnBFUVdoQ1oyOXlRbWRGUlVGWlR5OU5RVVZWUWtKTlRVVllaSFpqYlhSdFlrYzVNMWd5VW5Cak0wSm9aRWRPYjAxSFVVZERhWE5IUVZGUlFncG5OemgzUVZKVlJWWm5lRlZoU0ZJd1kwaE5Oa3g1T1c1aFdGSnZaRmRKZFZreU9YUk1NbWd4V2pKa2NHSnRaRzFaVjA1c1RESjBiR050Tld4aVNFMTBDbGt5T1hSaVdGWjFZVmhTTlV3eVJtcGtSMngyWW01TmRtTnVWblZqZVRoNlQxUkplazlVVlhoUFJFVjNUa001YUdSSVVteGlXRUl3WTNrNGVFMUNXVWNLUTJselIwRlJVVUpuTnpoM1FWSlpSVU5CZDBkalNGWnBZa2RzYWsxRldVZERhWE5IUVZGUlFtYzNPSGRCVW1kRlQwRjNNbU50Vm5kaWVuQnZaRmRrYmdwaFZ6VnVXbTFHYWxwVE9YSmFXRXAxV2xkNGVreFhUblppVnpFeFltMXNNR1ZVY0hsYVYxazJZMjFXYldONU9XOWFWMFpyWTNrNWRGbFhiSFZOU1VkTUNrSm5iM0pDWjBWRlFXUmFOVUZuVVVOQ1NEQkZaWGRDTlVGSVkwRXpWREIzWVhOaVNFVlVTbXBIVWpSamJWZGpNMEZ4U2t0WWNtcGxVRXN6TDJnMGNIa0taME00Y0Rkdk5FRkJRVWRvVm10d1JuTm5RVUZDUVUxQlUwUkNSMEZwUlVGek9XTlRVREJRTDNCU1dqUndjbUZpVGs1SmNUZzVWRk13VW1oWU1tWXpTZ3BTY0VoeWVXVkphRkl6V1VOSlVVUnVVWFYxV0hoVGJtNXVabXc0VVRkVlVrSlROMFJETWpOWmNrUmhVM0J6YUZOU1ZVVk9XSFJvVTJWNlFVdENaMmR4Q21ocmFrOVFVVkZFUVhkT2JrRkVRbXRCYWtWQk5HZERRMlV2TURCRVMxSXZOemhHYTNOU2N6RXhRbWROVjFOVGMybHphemhOUlRaUFRYbDRNbHBQUzBrS2RrMVdlVEZXU0VreE5VVTJXR1p5TTJselFTdEJhVGxhU1RaUGRIWTJRVzVQVm5jdmVsVklTMHgyZFdvMlZtOVpaRXh3TWpaVWQyaFhXVTR5VGpoTVJncGFhbnBXUjA1WUt6Qk9ZMDQ0VHpaeVdXTlFWWE4zUFQwS0xTMHRMUzFGVGtRZ1EwVlNWRWxHU1VOQlZFVXRMUzB0TFFvPSJ9fX19"}], "timestampVerificationData":{"rfc3161Timestamps":[{"signedTimestamp":"MIICyDADAgEAMIICvwYJKoZIhvcNAQcCoIICsDCCAqwCAQMxDTALBglghkgBZQMEAgEwgbcGCyqGSIb3DQEJEAEEoIGnBIGkMIGhAgEBBgkrBgEEAYO/MAIwMTANBglghkgBZQMEAgEFAAQgVRiPQiyjqHesLhjA1G/TxjESiy/FW8Sle/1OYNu3K/kCFAwvXO0pPiaW4uamLJqBsr3bixmnGA8yMDI2MTAxOTIyMzExMFowAwIBAaAypDAwLjEVMBMGA1UEChMMc2lnc3RvcmUuZGV2MRUwEwYDVQQDEwxzaWdzdG9yZS10c2GgADGCAdowggHWAgEBMFEwOTEVMBMGA1UEChMMc2lnc3RvcmUuZGV2MSAwHgYDVQQDExdzaWdzdG9yZS10c2Etc2VsZnNpZ25lZAIUOhNULwyQYe68wUMvy4qOiyojiwwwCwYJYIZIAWUDBAIBoIH8MBoGCSqGSIb3DQEJAzENBgsqhkiG9w0BCRABBDAcBgkqhkiG9w0BCQUxDxcNMjYxMDE5MjIzMTEwWjAvBgkqhkiG9w0BCQQxIgQgMtOdPWovwN2MtCUsGr55eXS/sj80XwnkzhWzvp3Z+ZYwgY4GCyqGSIb3DQEJEAIvMX8wfTB7MHkEIIX5J7wHq2LKw7RDVsEO/IGyxog/2nq55thw2dE6zQW3MFUwPaQ7MDkxFTATBgNVBAoTDHNpZ3N0b3JlLmRldjEgMB4GA1UEAxMXc2lnc3RvcmUtdHNhLXNlbGZzaWduZWQCFDoTVC8MkGHuvMFDL8uKjosqI4sMMAoGCCqGSM49BAMCBGYwZAIwa5hPScsoY8ZaoBk5cLEAUwhE+yEEOwm9xprfopTKz7qKFY1O9DeK5h6/gp2k6NtgAjByqi8mda8A+OGfG6rtkFHgjYnT6P/VUqQ95DNedeBl+vbMOity7vq28Gh+8O2yqBc="}]}}, "messageSignature":{"messageDigest":{"algorithm":"SHA2_256", "digest":"CPoM1QqE+UEnbF5tX6SfHU4B4VmHpgqrjyq24xnux3s="}, "signature":"MEUCICPSiaX9pY/oZbzuasGAnOQ0uL8P/JLHK+FKmUHYfvT2AiEAs8FC8HRyVEfj7AGjbsI6VfoahccKXs4FCfcyngTsQXc="}}
build/torch210-rocm70-x86_64-linux/mha.py ADDED
@@ -0,0 +1,332 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FlashAttention forward entry points backed by aiter's Composable-Kernel FMHA.
2
+
3
+ This is the compiled (HIP / Composable Kernel) counterpart to the Triton
4
+ ``aiter-flash-attn`` package. Only the *forward* pass is provided (dense and
5
+ variable-length), including support for learnable attention sinks via
6
+ ``window_size[2]`` / ``s_aux`` (e.g. gpt-oss). The op set ships exclusively
7
+ the CK ``mha_fwd`` / ``mha_varlen_fwd`` kernels, so the sink path is always
8
+ taken (no fall-through to a non-sink ASM kernel).
9
+ """
10
+
11
+ from typing import Optional, Tuple, Union
12
+
13
+ import torch
14
+ from torch import Tensor
15
+
16
+ from ._ops import ops
17
+
18
+
19
+ def _round_multiple(x: int, m: int) -> int:
20
+ return (x + m - 1) // m * m
21
+
22
+
23
+ def _maybe_pad_headdim(t: Tensor, target: int) -> Tensor:
24
+ og = t.size(-1)
25
+ if og == target:
26
+ return t
27
+ return torch.nn.functional.pad(t, [0, target - og])
28
+
29
+
30
+ def _unpack_window(window_size) -> Tuple[int, int, int]:
31
+ left = int(window_size[0])
32
+ right = int(window_size[1])
33
+ sink = int(window_size[2]) if len(window_size) == 3 else 0
34
+ return left, right, sink
35
+
36
+
37
+ def _prep_sink(s_aux: Optional[Tensor], nheads: int, device) -> Optional[Tensor]:
38
+ if s_aux is None:
39
+ return None
40
+ assert s_aux.device == device, "s_aux must be on the same device as q"
41
+ assert s_aux.shape[0] == nheads, "s_aux must have shape [nheads_q]"
42
+ if s_aux.dtype != torch.float32:
43
+ s_aux = s_aux.to(torch.float32)
44
+ return s_aux
45
+
46
+
47
+ def _effective_sink_size(sink_size: int, s_aux: Optional[Tensor]) -> int:
48
+ # The CK kernel only takes the sink path when sink_size > 0. Callers using
49
+ # the transformers attention interface pass the per-head sink logits as
50
+ # ``s_aux`` but leave ``window_size`` a 2-tuple (sink_size == 0), so enable
51
+ # a single learnable sink automatically when s_aux is given.
52
+ if s_aux is not None and sink_size == 0:
53
+ return 1
54
+ return sink_size
55
+
56
+
57
+ def flash_attn_func(
58
+ q: Tensor,
59
+ k: Tensor,
60
+ v: Tensor,
61
+ dropout_p: float = 0.0,
62
+ softmax_scale: Optional[float] = None,
63
+ causal: bool = False,
64
+ window_size: Tuple[int, ...] = (-1, -1, 0),
65
+ bias: Optional[Tensor] = None,
66
+ alibi_slopes: Optional[Tensor] = None,
67
+ return_lse: bool = False,
68
+ return_attn_probs: bool = False,
69
+ s_aux: Optional[Tensor] = None,
70
+ **kwargs,
71
+ ) -> Union[Tensor, Tuple[Tensor, ...]]:
72
+ """Dense FlashAttention forward.
73
+
74
+ ``q, k, v`` have shape ``(batch, seqlen, nheads, headdim)``. ``window_size``
75
+ is ``(left, right, sink_size)``; ``s_aux`` is an optional ``(nheads_q,)``
76
+ fp32 tensor of learnable sink logits. Extra keyword arguments (e.g. those
77
+ passed by the transformers flash-attention interface) are ignored.
78
+ """
79
+ if softmax_scale is None:
80
+ softmax_scale = q.shape[-1] ** (-0.5)
81
+ window_size_left, window_size_right, sink_size = _unpack_window(window_size)
82
+
83
+ nheads_q = q.size(2)
84
+ s_aux = _prep_sink(s_aux, nheads_q, q.device)
85
+ sink_size = _effective_sink_size(sink_size, s_aux)
86
+
87
+ head_q_og = q.size(-1)
88
+ head_v_og = v.size(-1)
89
+ head_q = _round_multiple(head_q_og, 8)
90
+ head_v = _round_multiple(head_v_og, 8)
91
+ q = _maybe_pad_headdim(q, head_q)
92
+ k = _maybe_pad_headdim(k, head_q)
93
+ v = _maybe_pad_headdim(v, head_v)
94
+
95
+ return_softmax = return_attn_probs and dropout_p > 0
96
+ out, softmax_lse, S_dmask, _ = ops.mha_fwd(
97
+ q,
98
+ k,
99
+ v,
100
+ dropout_p,
101
+ softmax_scale,
102
+ causal,
103
+ window_size_left,
104
+ window_size_right,
105
+ sink_size,
106
+ return_lse or return_attn_probs,
107
+ return_softmax,
108
+ None, # cu_seqlens_q
109
+ None, # cu_seqlens_kv
110
+ None, # out
111
+ bias,
112
+ alibi_slopes,
113
+ None, # q_descale
114
+ None, # k_descale
115
+ None, # v_descale
116
+ s_aux,
117
+ None, # gen
118
+ )
119
+
120
+ out = out[..., :head_v_og]
121
+ if not (return_lse or return_attn_probs):
122
+ return out
123
+ result = (out,)
124
+ if return_lse:
125
+ result = result + (softmax_lse,)
126
+ if return_attn_probs:
127
+ result = result + (S_dmask,)
128
+ return result
129
+
130
+
131
+ def flash_attn_varlen_func(
132
+ q: Tensor,
133
+ k: Tensor,
134
+ v: Tensor,
135
+ cu_seqlens_q: Tensor,
136
+ cu_seqlens_k: Tensor,
137
+ max_seqlen_q: int,
138
+ max_seqlen_k: int,
139
+ dropout_p: float = 0.0,
140
+ softmax_scale: Optional[float] = None,
141
+ causal: bool = False,
142
+ window_size: Tuple[int, ...] = (-1, -1, 0),
143
+ bias: Optional[Tensor] = None,
144
+ alibi_slopes: Optional[Tensor] = None,
145
+ return_lse: bool = False,
146
+ return_attn_probs: bool = False,
147
+ block_table: Optional[Tensor] = None,
148
+ min_seqlen_q: int = 0,
149
+ logits_soft_cap: float = 0.0,
150
+ zero_tensors: bool = False,
151
+ cu_seqlens_q_padded: Optional[Tensor] = None,
152
+ cu_seqlens_k_padded: Optional[Tensor] = None,
153
+ s_aux: Optional[Tensor] = None,
154
+ **kwargs,
155
+ ) -> Union[Tensor, Tuple[Tensor, ...]]:
156
+ """Variable-length / packed FlashAttention forward.
157
+
158
+ ``q`` has shape ``(total_q, nheads, headdim)``; ``cu_seqlens_*`` are int32
159
+ cumulative offsets of shape ``(batch + 1,)``. Sinks are supported through
160
+ ``window_size[2]`` / ``s_aux`` as in :func:`flash_attn_func`. Extra keyword
161
+ arguments (e.g. from the transformers flash-attention interface) are ignored.
162
+ """
163
+ if softmax_scale is None:
164
+ softmax_scale = q.shape[-1] ** (-0.5)
165
+ window_size_left, window_size_right, sink_size = _unpack_window(window_size)
166
+
167
+ nheads_q = q.size(1)
168
+ s_aux = _prep_sink(s_aux, nheads_q, q.device)
169
+ sink_size = _effective_sink_size(sink_size, s_aux)
170
+
171
+ head_q_og = q.size(-1)
172
+ head_v_og = v.size(-1)
173
+ head_q = _round_multiple(head_q_og, 8)
174
+ head_v = _round_multiple(head_v_og, 8)
175
+ q = _maybe_pad_headdim(q, head_q)
176
+ k = _maybe_pad_headdim(k, head_q)
177
+ v = _maybe_pad_headdim(v, head_v)
178
+
179
+ return_softmax = return_attn_probs and dropout_p > 0
180
+ out, softmax_lse, S_dmask, _ = ops.mha_varlen_fwd(
181
+ q,
182
+ k,
183
+ v,
184
+ cu_seqlens_q,
185
+ cu_seqlens_k,
186
+ max_seqlen_q,
187
+ max_seqlen_k,
188
+ min_seqlen_q,
189
+ dropout_p,
190
+ softmax_scale,
191
+ logits_soft_cap,
192
+ zero_tensors,
193
+ causal,
194
+ window_size_left,
195
+ window_size_right,
196
+ sink_size,
197
+ return_lse or return_attn_probs,
198
+ return_softmax,
199
+ None, # out
200
+ block_table,
201
+ bias,
202
+ alibi_slopes,
203
+ None, # q_descale
204
+ None, # k_descale
205
+ None, # v_descale
206
+ None, # gen
207
+ cu_seqlens_q_padded,
208
+ cu_seqlens_k_padded,
209
+ s_aux,
210
+ )
211
+
212
+ out = out[..., :head_v_og]
213
+ if not (return_lse or return_attn_probs):
214
+ return out
215
+ result = (out,)
216
+ if return_lse:
217
+ result = result + (softmax_lse,)
218
+ if return_attn_probs:
219
+ result = result + (S_dmask,)
220
+ return result
221
+
222
+
223
+ def _append_to_contiguous_cache(k_cache, v_cache, k, v, cache_seqlens, Sq):
224
+ """In-place append of new k/v into a contiguous cache at per-batch offsets."""
225
+ B = k_cache.shape[0]
226
+ if isinstance(cache_seqlens, int):
227
+ cache_seqlens = torch.full((B,), cache_seqlens, device=k_cache.device, dtype=torch.long)
228
+ pos = cache_seqlens.view(B, 1).to(torch.long) + torch.arange(Sq, device=k_cache.device).view(1, Sq)
229
+ bidx = torch.arange(B, device=k_cache.device).view(B, 1).expand(B, Sq)
230
+ k_cache[bidx, pos] = k.to(k_cache.dtype)
231
+ v_cache[bidx, pos] = v.to(v_cache.dtype)
232
+
233
+
234
+ def flash_attn_with_kvcache(
235
+ q: Tensor,
236
+ k_cache: Tensor,
237
+ v_cache: Tensor,
238
+ k: Optional[Tensor] = None,
239
+ v: Optional[Tensor] = None,
240
+ rotary_cos: Optional[Tensor] = None,
241
+ rotary_sin: Optional[Tensor] = None,
242
+ cache_seqlens=None,
243
+ block_table: Optional[Tensor] = None,
244
+ softmax_scale: Optional[float] = None,
245
+ causal: bool = True,
246
+ window_size: Tuple[int, ...] = (-1, -1, 0),
247
+ s_aux: Optional[Tensor] = None,
248
+ return_lse: bool = False,
249
+ **kwargs,
250
+ ) -> Union[Tensor, Tuple[Tensor, ...]]:
251
+ """FlashAttention against a KV cache, built on the CK split-KV kernel.
252
+
253
+ ``q`` is ``(batch, seqlen_q, nheads, headdim)``. For a contiguous cache,
254
+ ``k_cache``/``v_cache`` are ``(batch, seqlen_cache, nheads_k, headdim)`` and
255
+ ``cache_seqlens`` (int or ``(batch,)``) is the number of valid cache entries
256
+ *before* appending. If ``k``/``v`` (the new tokens) are given, they are
257
+ written into the cache in place. The query attends to the valid prefix
258
+ (``cache_seqlens`` + ``seqlen_q``) with bottom-right causal alignment.
259
+
260
+ Notes / current limitations:
261
+ - Rotary embedding is not applied here; pass ``q``/``k`` already rotated
262
+ (``rotary_cos``/``rotary_sin`` are rejected).
263
+ - Paged caches are addressed via ``block_table`` (forwarded to the split-KV
264
+ varlen kernel); in-cache append for paged layouts must be done by the
265
+ caller (pass ``k=v=None``).
266
+ """
267
+ if rotary_cos is not None or rotary_sin is not None:
268
+ raise NotImplementedError(
269
+ "flash_attn_with_kvcache does not apply rotary; rotate q/k beforehand."
270
+ )
271
+ if softmax_scale is None:
272
+ softmax_scale = q.shape[-1] ** (-0.5)
273
+ window_size_left, window_size_right, sink_size = _unpack_window(window_size)
274
+ B, Sq, Hq, D = q.shape
275
+ s_aux = _prep_sink(s_aux, Hq, q.device)
276
+ sink_size = _effective_sink_size(sink_size, s_aux)
277
+
278
+ paged = block_table is not None
279
+
280
+ # ---- append new tokens into the cache (contiguous layout only) ----
281
+ if k is not None and not paged:
282
+ assert cache_seqlens is not None, "cache_seqlens required to append into the cache"
283
+ _append_to_contiguous_cache(k_cache, v_cache, k, v, cache_seqlens, Sq)
284
+ elif k is not None and paged:
285
+ raise NotImplementedError(
286
+ "in-cache append for paged layouts is not supported; append externally "
287
+ "and call with k=v=None."
288
+ )
289
+
290
+ appended = 0 if k is None else Sq
291
+
292
+ # ---- key lengths ----
293
+ if cache_seqlens is None:
294
+ klens = torch.full((B,), k_cache.shape[1], device=q.device, dtype=torch.long)
295
+ elif isinstance(cache_seqlens, int):
296
+ klens = torch.full((B,), cache_seqlens + appended, device=q.device, dtype=torch.long)
297
+ else:
298
+ klens = cache_seqlens.to(q.device, torch.long) + appended
299
+
300
+ uniform = bool((klens == klens[0]).all().item())
301
+
302
+ # ---- fast path: contiguous cache, uniform length -> dense kernel ----
303
+ if not paged and uniform:
304
+ L = int(klens[0].item())
305
+ return flash_attn_func(
306
+ q, k_cache[:, :L], v_cache[:, :L],
307
+ softmax_scale=softmax_scale, causal=causal,
308
+ window_size=(window_size_left, window_size_right, sink_size),
309
+ s_aux=s_aux, return_lse=return_lse,
310
+ )
311
+
312
+ # ---- general path: variable cache lengths -> split-KV varlen kernel ----
313
+ cu_q = torch.arange(0, (B + 1) * Sq, Sq, device=q.device, dtype=torch.int32)
314
+ cu_k = torch.zeros(B + 1, device=q.device, dtype=torch.int32)
315
+ cu_k[1:] = torch.cumsum(klens, 0).to(torch.int32)
316
+ q_packed = q.reshape(B * Sq, Hq, D)
317
+ if paged:
318
+ k_packed, v_packed = k_cache, v_cache # kernel gathers via block_table
319
+ else:
320
+ k_packed = torch.cat([k_cache[b, : int(klens[b])] for b in range(B)], dim=0)
321
+ v_packed = torch.cat([v_cache[b, : int(klens[b])] for b in range(B)], dim=0)
322
+
323
+ out = flash_attn_varlen_func(
324
+ q_packed, k_packed, v_packed, cu_q, cu_k, Sq, int(klens.max().item()),
325
+ softmax_scale=softmax_scale, causal=causal,
326
+ window_size=(window_size_left, window_size_right, sink_size),
327
+ block_table=block_table, s_aux=s_aux, return_lse=return_lse,
328
+ )
329
+ if return_lse:
330
+ out, lse = out
331
+ return out.reshape(B, Sq, Hq, D), lse
332
+ return out.reshape(B, Sq, Hq, D)
build/torch210-rocm70-x86_64-linux/symbols.json ADDED
@@ -0,0 +1,349 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "schema_version": 1,
3
+ "module": "_aiter_flash_attn_ck_rocm_fe2bb66",
4
+ "functions": [
5
+ {
6
+ "name": "flash_attn_func",
7
+ "qualname": "flash_attn_func",
8
+ "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha",
9
+ "kind": "function",
10
+ "doc": "Dense FlashAttention forward.\n\n``q, k, v`` have shape ``(batch, seqlen, nheads, headdim)``. ``window_size``\nis ``(left, right, sink_size)``; ``s_aux`` is an optional ``(nheads_q,)``\nfp32 tensor of learnable sink logits. Extra keyword arguments (e.g. those\npassed by the transformers flash-attention interface) are ignored.",
11
+ "signature": {
12
+ "parameters": [
13
+ {
14
+ "name": "q",
15
+ "kind": "POSITIONAL_OR_KEYWORD",
16
+ "default": null,
17
+ "annotation": "torch.Tensor"
18
+ },
19
+ {
20
+ "name": "k",
21
+ "kind": "POSITIONAL_OR_KEYWORD",
22
+ "default": null,
23
+ "annotation": "torch.Tensor"
24
+ },
25
+ {
26
+ "name": "v",
27
+ "kind": "POSITIONAL_OR_KEYWORD",
28
+ "default": null,
29
+ "annotation": "torch.Tensor"
30
+ },
31
+ {
32
+ "name": "dropout_p",
33
+ "kind": "POSITIONAL_OR_KEYWORD",
34
+ "default": "0.0",
35
+ "annotation": "float"
36
+ },
37
+ {
38
+ "name": "softmax_scale",
39
+ "kind": "POSITIONAL_OR_KEYWORD",
40
+ "default": "None",
41
+ "annotation": "float | None"
42
+ },
43
+ {
44
+ "name": "causal",
45
+ "kind": "POSITIONAL_OR_KEYWORD",
46
+ "default": "False",
47
+ "annotation": "bool"
48
+ },
49
+ {
50
+ "name": "window_size",
51
+ "kind": "POSITIONAL_OR_KEYWORD",
52
+ "default": "(-1, -1, 0)",
53
+ "annotation": "Tuple[int, ...]"
54
+ },
55
+ {
56
+ "name": "bias",
57
+ "kind": "POSITIONAL_OR_KEYWORD",
58
+ "default": "None",
59
+ "annotation": "torch.Tensor | None"
60
+ },
61
+ {
62
+ "name": "alibi_slopes",
63
+ "kind": "POSITIONAL_OR_KEYWORD",
64
+ "default": "None",
65
+ "annotation": "torch.Tensor | None"
66
+ },
67
+ {
68
+ "name": "return_lse",
69
+ "kind": "POSITIONAL_OR_KEYWORD",
70
+ "default": "False",
71
+ "annotation": "bool"
72
+ },
73
+ {
74
+ "name": "return_attn_probs",
75
+ "kind": "POSITIONAL_OR_KEYWORD",
76
+ "default": "False",
77
+ "annotation": "bool"
78
+ },
79
+ {
80
+ "name": "s_aux",
81
+ "kind": "POSITIONAL_OR_KEYWORD",
82
+ "default": "None",
83
+ "annotation": "torch.Tensor | None"
84
+ },
85
+ {
86
+ "name": "kwargs",
87
+ "kind": "VAR_KEYWORD",
88
+ "default": null,
89
+ "annotation": null
90
+ }
91
+ ],
92
+ "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]"
93
+ }
94
+ },
95
+ {
96
+ "name": "flash_attn_varlen_func",
97
+ "qualname": "flash_attn_varlen_func",
98
+ "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha",
99
+ "kind": "function",
100
+ "doc": "Variable-length / packed FlashAttention forward.\n\n``q`` has shape ``(total_q, nheads, headdim)``; ``cu_seqlens_*`` are int32\ncumulative offsets of shape ``(batch + 1,)``. Sinks are supported through\n``window_size[2]`` / ``s_aux`` as in :func:`flash_attn_func`. Extra keyword\narguments (e.g. from the transformers flash-attention interface) are ignored.",
101
+ "signature": {
102
+ "parameters": [
103
+ {
104
+ "name": "q",
105
+ "kind": "POSITIONAL_OR_KEYWORD",
106
+ "default": null,
107
+ "annotation": "torch.Tensor"
108
+ },
109
+ {
110
+ "name": "k",
111
+ "kind": "POSITIONAL_OR_KEYWORD",
112
+ "default": null,
113
+ "annotation": "torch.Tensor"
114
+ },
115
+ {
116
+ "name": "v",
117
+ "kind": "POSITIONAL_OR_KEYWORD",
118
+ "default": null,
119
+ "annotation": "torch.Tensor"
120
+ },
121
+ {
122
+ "name": "cu_seqlens_q",
123
+ "kind": "POSITIONAL_OR_KEYWORD",
124
+ "default": null,
125
+ "annotation": "torch.Tensor"
126
+ },
127
+ {
128
+ "name": "cu_seqlens_k",
129
+ "kind": "POSITIONAL_OR_KEYWORD",
130
+ "default": null,
131
+ "annotation": "torch.Tensor"
132
+ },
133
+ {
134
+ "name": "max_seqlen_q",
135
+ "kind": "POSITIONAL_OR_KEYWORD",
136
+ "default": null,
137
+ "annotation": "int"
138
+ },
139
+ {
140
+ "name": "max_seqlen_k",
141
+ "kind": "POSITIONAL_OR_KEYWORD",
142
+ "default": null,
143
+ "annotation": "int"
144
+ },
145
+ {
146
+ "name": "dropout_p",
147
+ "kind": "POSITIONAL_OR_KEYWORD",
148
+ "default": "0.0",
149
+ "annotation": "float"
150
+ },
151
+ {
152
+ "name": "softmax_scale",
153
+ "kind": "POSITIONAL_OR_KEYWORD",
154
+ "default": "None",
155
+ "annotation": "float | None"
156
+ },
157
+ {
158
+ "name": "causal",
159
+ "kind": "POSITIONAL_OR_KEYWORD",
160
+ "default": "False",
161
+ "annotation": "bool"
162
+ },
163
+ {
164
+ "name": "window_size",
165
+ "kind": "POSITIONAL_OR_KEYWORD",
166
+ "default": "(-1, -1, 0)",
167
+ "annotation": "Tuple[int, ...]"
168
+ },
169
+ {
170
+ "name": "bias",
171
+ "kind": "POSITIONAL_OR_KEYWORD",
172
+ "default": "None",
173
+ "annotation": "torch.Tensor | None"
174
+ },
175
+ {
176
+ "name": "alibi_slopes",
177
+ "kind": "POSITIONAL_OR_KEYWORD",
178
+ "default": "None",
179
+ "annotation": "torch.Tensor | None"
180
+ },
181
+ {
182
+ "name": "return_lse",
183
+ "kind": "POSITIONAL_OR_KEYWORD",
184
+ "default": "False",
185
+ "annotation": "bool"
186
+ },
187
+ {
188
+ "name": "return_attn_probs",
189
+ "kind": "POSITIONAL_OR_KEYWORD",
190
+ "default": "False",
191
+ "annotation": "bool"
192
+ },
193
+ {
194
+ "name": "block_table",
195
+ "kind": "POSITIONAL_OR_KEYWORD",
196
+ "default": "None",
197
+ "annotation": "torch.Tensor | None"
198
+ },
199
+ {
200
+ "name": "min_seqlen_q",
201
+ "kind": "POSITIONAL_OR_KEYWORD",
202
+ "default": "0",
203
+ "annotation": "int"
204
+ },
205
+ {
206
+ "name": "logits_soft_cap",
207
+ "kind": "POSITIONAL_OR_KEYWORD",
208
+ "default": "0.0",
209
+ "annotation": "float"
210
+ },
211
+ {
212
+ "name": "zero_tensors",
213
+ "kind": "POSITIONAL_OR_KEYWORD",
214
+ "default": "False",
215
+ "annotation": "bool"
216
+ },
217
+ {
218
+ "name": "cu_seqlens_q_padded",
219
+ "kind": "POSITIONAL_OR_KEYWORD",
220
+ "default": "None",
221
+ "annotation": "torch.Tensor | None"
222
+ },
223
+ {
224
+ "name": "cu_seqlens_k_padded",
225
+ "kind": "POSITIONAL_OR_KEYWORD",
226
+ "default": "None",
227
+ "annotation": "torch.Tensor | None"
228
+ },
229
+ {
230
+ "name": "s_aux",
231
+ "kind": "POSITIONAL_OR_KEYWORD",
232
+ "default": "None",
233
+ "annotation": "torch.Tensor | None"
234
+ },
235
+ {
236
+ "name": "kwargs",
237
+ "kind": "VAR_KEYWORD",
238
+ "default": null,
239
+ "annotation": null
240
+ }
241
+ ],
242
+ "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]"
243
+ }
244
+ },
245
+ {
246
+ "name": "flash_attn_with_kvcache",
247
+ "qualname": "flash_attn_with_kvcache",
248
+ "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha",
249
+ "kind": "function",
250
+ "doc": "FlashAttention against a KV cache, built on the CK split-KV kernel.\n\n``q`` is ``(batch, seqlen_q, nheads, headdim)``. For a contiguous cache,\n``k_cache``/``v_cache`` are ``(batch, seqlen_cache, nheads_k, headdim)`` and\n``cache_seqlens`` (int or ``(batch,)``) is the number of valid cache entries\n*before* appending. If ``k``/``v`` (the new tokens) are given, they are\nwritten into the cache in place. The query attends to the valid prefix\n(``cache_seqlens`` + ``seqlen_q``) with bottom-right causal alignment.\n\nNotes / current limitations:\n- Rotary embedding is not applied here; pass ``q``/``k`` already rotated\n (``rotary_cos``/``rotary_sin`` are rejected).\n- Paged caches are addressed via ``block_table`` (forwarded to the split-KV\n varlen kernel); in-cache append for paged layouts must be done by the\n caller (pass ``k=v=None``).",
251
+ "signature": {
252
+ "parameters": [
253
+ {
254
+ "name": "q",
255
+ "kind": "POSITIONAL_OR_KEYWORD",
256
+ "default": null,
257
+ "annotation": "torch.Tensor"
258
+ },
259
+ {
260
+ "name": "k_cache",
261
+ "kind": "POSITIONAL_OR_KEYWORD",
262
+ "default": null,
263
+ "annotation": "torch.Tensor"
264
+ },
265
+ {
266
+ "name": "v_cache",
267
+ "kind": "POSITIONAL_OR_KEYWORD",
268
+ "default": null,
269
+ "annotation": "torch.Tensor"
270
+ },
271
+ {
272
+ "name": "k",
273
+ "kind": "POSITIONAL_OR_KEYWORD",
274
+ "default": "None",
275
+ "annotation": "torch.Tensor | None"
276
+ },
277
+ {
278
+ "name": "v",
279
+ "kind": "POSITIONAL_OR_KEYWORD",
280
+ "default": "None",
281
+ "annotation": "torch.Tensor | None"
282
+ },
283
+ {
284
+ "name": "rotary_cos",
285
+ "kind": "POSITIONAL_OR_KEYWORD",
286
+ "default": "None",
287
+ "annotation": "torch.Tensor | None"
288
+ },
289
+ {
290
+ "name": "rotary_sin",
291
+ "kind": "POSITIONAL_OR_KEYWORD",
292
+ "default": "None",
293
+ "annotation": "torch.Tensor | None"
294
+ },
295
+ {
296
+ "name": "cache_seqlens",
297
+ "kind": "POSITIONAL_OR_KEYWORD",
298
+ "default": "None",
299
+ "annotation": null
300
+ },
301
+ {
302
+ "name": "block_table",
303
+ "kind": "POSITIONAL_OR_KEYWORD",
304
+ "default": "None",
305
+ "annotation": "torch.Tensor | None"
306
+ },
307
+ {
308
+ "name": "softmax_scale",
309
+ "kind": "POSITIONAL_OR_KEYWORD",
310
+ "default": "None",
311
+ "annotation": "float | None"
312
+ },
313
+ {
314
+ "name": "causal",
315
+ "kind": "POSITIONAL_OR_KEYWORD",
316
+ "default": "True",
317
+ "annotation": "bool"
318
+ },
319
+ {
320
+ "name": "window_size",
321
+ "kind": "POSITIONAL_OR_KEYWORD",
322
+ "default": "(-1, -1, 0)",
323
+ "annotation": "Tuple[int, ...]"
324
+ },
325
+ {
326
+ "name": "s_aux",
327
+ "kind": "POSITIONAL_OR_KEYWORD",
328
+ "default": "None",
329
+ "annotation": "torch.Tensor | None"
330
+ },
331
+ {
332
+ "name": "return_lse",
333
+ "kind": "POSITIONAL_OR_KEYWORD",
334
+ "default": "False",
335
+ "annotation": "bool"
336
+ },
337
+ {
338
+ "name": "kwargs",
339
+ "kind": "VAR_KEYWORD",
340
+ "default": null,
341
+ "annotation": null
342
+ }
343
+ ],
344
+ "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]"
345
+ }
346
+ }
347
+ ],
348
+ "layers": []
349
+ }
build/torch210-rocm71-x86_64-linux/__init__.py ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """AITER Composable-Kernel FlashAttention kernels for AMD ROCm (compiled HIP).
2
+
3
+ Compiled (Composable Kernel / HIP) counterpart to the Triton-based
4
+ ``aiter-flash-attn`` package, repackaged from the CK FMHA forward path of the
5
+ `ROCm/aiter <https://github.com/ROCm/aiter>`_ project as a self-contained
6
+ Hugging Face Hub kernel. Exposes the FlashAttention forward entry points with
7
+ learnable attention sink support (e.g. gpt-oss).
8
+ """
9
+
10
+ from .mha import (
11
+ flash_attn_func,
12
+ flash_attn_varlen_func,
13
+ flash_attn_with_kvcache,
14
+ )
15
+
16
+
17
+ __kernel_metadata__ = {
18
+ "license": "mit",
19
+ }
20
+
21
+
22
+ __all__ = [
23
+ "__kernel_metadata__",
24
+ "flash_attn_func",
25
+ "flash_attn_varlen_func",
26
+ "flash_attn_with_kvcache",
27
+ ]
build/torch210-rocm71-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:04aebc20ca3de88cd0cb137fba3aaf0aa8b1ccd16588df660c70c8491653f01c
3
+ size 204259752
build/torch210-rocm71-x86_64-linux/_ops.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _aiter_flash_attn_ck_rocm_fe2bb66
3
+ ops = torch.ops._aiter_flash_attn_ck_rocm_fe2bb66
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ """
7
+ Prefix op by namespace.
8
+ """
9
+ return f"_aiter_flash_attn_ck_rocm_fe2bb66::{op_name}"
build/torch210-rocm71-x86_64-linux/metadata.json ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "aiter-flash-attn-ck",
3
+ "id": "_aiter_flash_attn_ck_rocm_fe2bb66",
4
+ "version": 1,
5
+ "kernels-minver": "0.14.0",
6
+ "license": "MIT",
7
+ "python-depends": [],
8
+ "kernel-depends": [],
9
+ "backend": {
10
+ "type": "rocm",
11
+ "archs": [
12
+ "gfx942",
13
+ "gfx950"
14
+ ]
15
+ },
16
+ "digest": {
17
+ "algorithm": "sha256",
18
+ "files": {
19
+ "__init__.py": "7wj1GTm2bqTLLCyT4xLuAk/+JWYrhINjY/ODMNWhgZ0=",
20
+ "_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so": "BK68IMo96IzQyxN/ujqvCqixzNFliN9mDHDISRZT8Bw=",
21
+ "_ops.py": "4WkdYDWWJp60zep+oW/5ym9vIeVMNfrgRCkP3o/w7yU=",
22
+ "mha.py": "zCAjY4O+DiCzmEL+W9sqXxZaqjsj4RJbWFcyPpUZejk=",
23
+ "symbols.json": "rhnF4H8Jsb6I+YPxIVFMgnLcTQJODEXbZ7ge77NZXAU="
24
+ }
25
+ },
26
+ "provenance": {
27
+ "kernel-builder": {
28
+ "version": "0.18.0-dev0",
29
+ "commit": "d054906a13710b811b96697b4635f03020842733",
30
+ "dirty": false
31
+ },
32
+ "kernel": {
33
+ "commit": "fe2bb66fee6c2a77eccf08a0508dc57428d488ed",
34
+ "dirty": false
35
+ }
36
+ }
37
+ }
build/torch210-rocm71-x86_64-linux/metadata.json.sigstore ADDED
@@ -0,0 +1 @@
 
 
1
+ {"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json", "verificationMaterial":{"certificate":{"rawBytes":"MIIHLDCCBrKgAwIBAgIUMp+DSFw+JgfWV7TcLB0c0Mb65MwwCgYIKoZIzj0EAwMwNzEVMBMGA1UEChMMc2lnc3RvcmUuZGV2MR4wHAYDVQQDExVzaWdzdG9yZS1pbnRlcm1lZGlhdGUwHhcNMjYxMDE5MjIzMTExWhcNMjYxMDE5MjI0MTExWjAAMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE8kLbyyRJUo+k0QdJbUOn+ybnTHhtfKEr7dpiN7ICGRBSv7puurajV7kDVk6RJbao8Okhui99gTQi7oGPT0Nxt6OCBdEwggXNMA4GA1UdDwEB/wQEAwIHgDATBgNVHSUEDDAKBggrBgEFBQcDAzAfBgNVHSMEGDAWgBTf0+nPViQRlvmo2OkoVaLGLhhkPzBrBgNVHREBAf8EYTBfhl1odHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkvLmdpdGh1Yi93b3JrZmxvd3MvYnVpbGQueWFtbEByZWZzL2hlYWRzL21haW4wOQYKKwYBBAGDvzABAQQraHR0cHM6Ly90b2tlbi5hY3Rpb25zLmdpdGh1YnVzZXJjb250ZW50LmNvbTAfBgorBgEEAYO/MAECBBF3b3JrZmxvd19kaXNwYXRjaDA2BgorBgEEAYO/MAEDBChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMBMGCisGAQQBg78wAQQEBUJ1aWxkMCsGCisGAQQBg78wAQUEHWh1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5MB0GCisGAQQBg78wAQYED3JlZnMvaGVhZHMvbWFpbjA7BgorBgEEAYO/MAEIBC0MK2h0dHBzOi8vdG9rZW4uYWN0aW9ucy5naXRodWJ1c2VyY29udGVudC5jb20wbQYKKwYBBAGDvzABCQRfDF1odHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkvLmdpdGh1Yi93b3JrZmxvd3MvYnVpbGQueWFtbEByZWZzL2hlYWRzL21haW4wOAYKKwYBBAGDvzABCgQqDChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMBsGCisGAQQBg78wAQsEDQwLc2VsZi1ob3N0ZWQwQAYKKwYBBAGDvzABDAQyDDBodHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkwOAYKKwYBBAGDvzABDQQqDChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMB8GCisGAQQBg78wAQ4EEQwPcmVmcy9oZWFkcy9tYWluMBoGCisGAQQBg78wAQ8EDAwKMTA3MTQ3NTUyOTAuBgorBgEEAYO/MAEQBCAMHmh0dHBzOi8vZ2l0aHViLmNvbS9odWdnaW5nZmFjZTAYBgorBgEEAYO/MAERBAoMCDI1NzIwNzQzMG0GCisGAQQBg78wARIEXwxdaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5Ly5naXRodWIvd29ya2Zsb3dzL2J1aWxkLnlhbWxAcmVmcy9oZWFkcy9tYWluMDgGCisGAQQBg78wARMEKgwoZmUyYmI2NmZlZTZjMmE3N2VjY2YwOGEwNTA4ZGM1NzQyOGQ0ODhlZDAhBgorBgEEAYO/MAEUBBMMEXdvcmtmbG93X2Rpc3BhdGNoMGQGCisGAQQBg78wARUEVgxUaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5L2FjdGlvbnMvcnVucy8zOTIzOTUxODEwNC9hdHRlbXB0cy8xMBYGCisGAQQBg78wARYECAwGcHVibGljMEYGCisGAQQBg78wARgEOAw2cmVwbzpodWdnaW5nZmFjZS9rZXJuZWxzLWNvbW11bml0eTpyZWY6cmVmcy9oZWFkcy9tYWluMIGKBgorBgEEAdZ5AgQCBHwEegB4AHYA3T0wasbHETJjGR4cmWc3AqJKXrjePK3/h4pygC8p7o4AAAGhVkpJ0wAABAMARzBFAiBd9Y7k3OakcDXOUOtPyvjvTHZaUz/B/tF4M/lrAPbSkgIhAMS6DTW0k6oKIEa1SATYHGWoTmXaF3t6lEktxjLrj5OMMAoGCCqGSM49BAMDA2gAMGUCMH23PJfryGMZHm69vca7K+4CeSSdtIsPRohlbI62CVTQgx/5zELDp54UGy2s+s8O0AIxAIDnrnYiN4CaOuqRlp6WpPLQuG0Xvzk3XM/ioaBbCV9mGfH1y674/krGMefcljicUw=="}, "tlogEntries":[{"logIndex":"3327211242", "logId":{"keyId":"wNI9atQGlz+VWfO6LRygH4QUfY/8W4RFwiT5i5WRgB0="}, "kindVersion":{"kind":"hashedrekord", "version":"0.0.1"}, "integratedTime":"1792449071", "inclusionPromise":{"signedEntryTimestamp":"MEQCIAr6PySJYJqy9DPHTtQgj9lSQLP+7jzp1PQhaQdDBhOkAiATFyyPYAtt+8DMi8K3HCc98mwBas4Kn6GOtrmIo+u8cA=="}, "inclusionProof":{"logIndex":"3205306980", "rootHash":"/n2pIUbsEGrPvDVhc62xJVK1pGu6X6FIjXWKAB9B11s=", "treeSize":"3205307004", "hashes":["b8JPr2VEt72nRhnrDm3LGBUw57uu9LLjYUP7BrgM9j8=", "uj8iz9+habwy51518PtXayHwhXS1UVWHasW4Dw4HEv8=", "ZzHzDD4a8QdIuk81wimU52tqU/HqEaVUd5zE4szbH5k=", "hKyjtQs1OpKphopD5PuSJ+QLeWBbsxl1l8V8vcQPgug=", "GNln8dir9ny0f06ahhipF8UWPa4XL9o9Bf1LV9qfH9M=", "v5NV1PqmPcm+8RACz+834UW43Jvx9T42P+ziCOAZqEY=", "t5D+40j8Z6l/36YGQvBik1shdJk9qiXxj5J6yRy+Ilw=", "t4Yy4kaoiJ3+VosqXw5Ogmg+A7p/E6y8+JhqWbalMbI=", "5egUlZ4Pi0Ix/qLyCDXocsWywJCHqrde/S06vOZvlOM=", "QsXufUpoKtSjKAB19K1G00umQpWvPHlBNuRGFiiVjQQ=", "QIMkwQbwAewhLJd//5FFwI3SnyNQBGjCgJrelfG8vhc=", "Et17CACUv9QN64irUJuBjsrgRGkGnnJ9xacRV/aBdwQ=", "lqgEwkxhDMs/gdIAE4lmjfMwg7nDmv7lda6O0LGliCw=", "HovTPBWzoErvUg1u5sIi3x1ZonZ7yJekVsBrmpomCIA=", "G+nk7mgE5h6dUmv2UY7NDJrY3WvrZY31raaO1AETt+o=", "qlTWNDzERnTBQVcHHXXVPjiUP0SsikGxfcxVlgqVFDo=", "gg9GNApNke3+zNhl4tsWtmIycCFrOMCB3lys6XuSQNQ=", "NlOclldmKPA1tMKIskqHAwuQDYHp31YwpK+OOMrMccc=", "qxzHanAzz57SDdmJe0B7bJK72NTIBbwEMGBBKvDOROw=", "xH/DCseLHr9eKoYT8qsORZK7zVdEGYWHuVtsVrD95wY="], "checkpoint":{"envelope":"rekor.sigstore.dev - 1193050959916656506\n3205307004\n/n2pIUbsEGrPvDVhc62xJVK1pGu6X6FIjXWKAB9B11s=\n\n— rekor.sigstore.dev wNI9ajBFAiAbIQNvzdYi+kJoZke0K/Pqgv3/lAKAHbSNYL82CNBKXQIhAJl7oEPKWHot9DzcEzfhRkqZL/pbPqMksdsMLXmuh/4s\n"}}, "canonicalizedBody":"eyJhcGlWZXJzaW9uIjoiMC4wLjEiLCJraW5kIjoiaGFzaGVkcmVrb3JkIiwic3BlYyI6eyJkYXRhIjp7Imhhc2giOnsiYWxnb3JpdGhtIjoic2hhMjU2IiwidmFsdWUiOiI5MWY4ZDQxN2NiYzEyMDY4YjIyMGQwNmY4ZmYxNTg5MzBiNTdjZTc1ZjVjMzkzOTc5MTQwOTgwNzQ1YThmNTQyIn19LCJzaWduYXR1cmUiOnsiY29udGVudCI6Ik1FUUNJR0hqWFJuSmpUbnFaaVZ1QmpNZU9oYU54aHIwSFE2enpSckI0cVBlN1Y5M0FpQm9QeS9kT1FLcVZNOWlnTGpSbmhOeFFBQ1plLzM4YWZLSFBqMExNZk5NdUE9PSIsInB1YmxpY0tleSI6eyJjb250ZW50IjoiTFMwdExTMUNSVWRKVGlCRFJWSlVTVVpKUTBGVVJTMHRMUzB0Q2sxSlNVaE1SRU5EUW5KTFowRjNTVUpCWjBsVlRYQXJSRk5HZHl0S1oyWlhWamRVWTB4Q01HTXdUV0kyTlUxM2QwTm5XVWxMYjFwSmVtb3dSVUYzVFhjS1RucEZWazFDVFVkQk1WVkZRMmhOVFdNeWJHNWpNMUoyWTIxVmRWcEhWakpOVWpSM1NFRlpSRlpSVVVSRmVGWjZZVmRrZW1SSE9YbGFVekZ3WW01U2JBcGpiVEZzV2tkc2FHUkhWWGRJYUdOT1RXcFplRTFFUlRWTmFrbDZUVlJGZUZkb1kwNU5hbGw0VFVSRk5VMXFTVEJOVkVWNFYycEJRVTFHYTNkRmQxbElDa3R2V2tsNmFqQkRRVkZaU1V0dldrbDZhakJFUVZGalJGRm5RVVU0YTB4aWVYbFNTbFZ2SzJzd1VXUktZbFZQYml0NVltNVVTR2gwWmt0RmNqZGtjR2tLVGpkSlEwZFNRbE4yTjNCMWRYSmhhbFkzYTBSV2F6WlNTbUpoYnpoUGEyaDFhVGs1WjFSUmFUZHZSMUJVTUU1NGREWlBRMEprUlhkbloxaE9UVUUwUndwQk1WVmtSSGRGUWk5M1VVVkJkMGxJWjBSQlZFSm5UbFpJVTFWRlJFUkJTMEpuWjNKQ1owVkdRbEZqUkVGNlFXWkNaMDVXU0ZOTlJVZEVRVmRuUWxSbUNqQXJibEJXYVZGU2JIWnRiekpQYTI5V1lVeEhUR2hvYTFCNlFuSkNaMDVXU0ZKRlFrRm1PRVZaVkVKbWFHd3hiMlJJVW5kamVtOTJUREprY0dSSGFERUtXV2sxYW1JeU1IWmhTRlp1V2pKc2RWb3lXbWhaTWxWMllUSldlV0p0Vm5OamVURnFZakl4ZEdSWE5YQmtTR3QyVEcxa2NHUkhhREZaYVRrellqTktjZ3BhYlhoMlpETk5kbGx1Vm5CaVIxRjFaVmRHZEdKRlFubGFWMXA2VERKb2JGbFhVbnBNTWpGb1lWYzBkMDlSV1V0TGQxbENRa0ZIUkhaNlFVSkJVVkZ5Q21GSVVqQmpTRTAyVEhrNU1HSXlkR3hpYVRWb1dUTlNjR0l5TlhwTWJXUndaRWRvTVZsdVZucGFXRXBxWWpJMU1GcFhOVEJNYlU1MllsUkJaa0puYjNJS1FtZEZSVUZaVHk5TlFVVkRRa0pHTTJJelNuSmFiWGgyWkRFNWEyRllUbmRaV0ZKcVlVUkJNa0puYjNKQ1owVkZRVmxQTDAxQlJVUkNRMmh0V2xSS2FRcFphbGt5V20xV2JFNXRUWGxaVkdNeldsZE9hbHBxUVRSWlZFRXhUVVJvYTFsNlZUTk9SRWswV2tSUk5FOUhWbXROUWsxSFEybHpSMEZSVVVKbk56aDNDa0ZSVVVWQ1ZVb3hZVmQ0YTAxRGMwZERhWE5IUVZGUlFtYzNPSGRCVVZWRlNGZG9NVm95WkhCaWJXUnRXVmRPYkV3eWRHeGpiVFZzWWtoTmRGa3lPWFFLWWxoV2RXRllValZOUWpCSFEybHpSMEZSVVVKbk56aDNRVkZaUlVRelNteGFiazEyWVVkV2FGcElUWFppVjBad1ltcEJOMEpuYjNKQ1owVkZRVmxQTHdwTlFVVkpRa013VFVzeWFEQmtTRUo2VDJrNGRtUkhPWEphVnpSMVdWZE9NR0ZYT1hWamVUVnVZVmhTYjJSWFNqRmpNbFo1V1RJNWRXUkhWblZrUXpWcUNtSXlNSGRpVVZsTFMzZFpRa0pCUjBSMmVrRkNRMUZTWmtSR01XOWtTRkozWTNwdmRrd3laSEJrUjJneFdXazFhbUl5TUhaaFNGWnVXakpzZFZveVdtZ0tXVEpWZG1FeVZubGliVlp6WTNreGFtSXlNWFJrVnpWd1pFaHJka3h0WkhCa1IyZ3hXV2s1TTJJelNuSmFiWGgyWkROTmRsbHVWbkJpUjFGMVpWZEdkQXBpUlVKNVdsZGFla3d5YUd4WlYxSjZUREl4YUdGWE5IZFBRVmxMUzNkWlFrSkJSMFIyZWtGQ1EyZFJjVVJEYUcxYVZFcHBXV3BaTWxwdFZteE9iVTE1Q2xsVVl6TmFWMDVxV21wQk5GbFVRVEZOUkdocldYcFZNMDVFU1RSYVJGRTBUMGRXYTAxQ2MwZERhWE5IUVZGUlFtYzNPSGRCVVhORlJGRjNUR015Vm5NS1dta3hiMkl6VGpCYVYxRjNVVUZaUzB0M1dVSkNRVWRFZG5wQlFrUkJVWGxFUkVKdlpFaFNkMk42YjNaTU1tUndaRWRvTVZscE5XcGlNakIyWVVoV2JncGFNbXgxV2pKYWFGa3lWWFpoTWxaNVltMVdjMk41TVdwaU1qRjBaRmMxY0dSSWEzZFBRVmxMUzNkWlFrSkJSMFIyZWtGQ1JGRlJjVVJEYUcxYVZFcHBDbGxxV1RKYWJWWnNUbTFOZVZsVVl6TmFWMDVxV21wQk5GbFVRVEZOUkdocldYcFZNMDVFU1RSYVJGRTBUMGRXYTAxQ09FZERhWE5IUVZGUlFtYzNPSGNLUVZFMFJVVlJkMUJqYlZadFkzazViMXBYUm10amVUbDBXVmRzZFUxQ2IwZERhWE5IUVZGUlFtYzNPSGRCVVRoRlJFRjNTMDFVUVROTlZGRXpUbFJWZVFwUFZFRjFRbWR2Y2tKblJVVkJXVTh2VFVGRlVVSkRRVTFJYldnd1pFaENlazlwT0haYU1td3dZVWhXYVV4dFRuWmlVemx2WkZka2JtRlhOVzVhYlVacUNscFVRVmxDWjI5eVFtZEZSVUZaVHk5TlFVVlNRa0Z2VFVORVNURk9la2wzVG5wUmVrMUhNRWREYVhOSFFWRlJRbWMzT0hkQlVrbEZXSGQ0WkdGSVVqQUtZMGhOTmt4NU9XNWhXRkp2WkZkSmRWa3lPWFJNTW1neFdqSmtjR0p0WkcxWlYwNXNUREowYkdOdE5XeGlTRTEwV1RJNWRHSllWblZoV0ZJMVRIazFiZ3BoV0ZKdlpGZEpkbVF5T1hsaE1scHpZak5rZWt3eVNqRmhWM2hyVEc1c2FHSlhlRUZqYlZadFkzazViMXBYUm10amVUbDBXVmRzZFUxRVowZERhWE5IQ2tGUlVVSm5OemgzUVZKTlJVdG5kMjlhYlZWNVdXMUpNazV0V214YVZGcHFUVzFGTTA0eVZtcFpNbGwzVDBkRmQwNVVRVFJhUjAweFRucFJlVTlIVVRBS1QwUm9iRnBFUVdoQ1oyOXlRbWRGUlVGWlR5OU5RVVZWUWtKTlRVVllaSFpqYlhSdFlrYzVNMWd5VW5Cak0wSm9aRWRPYjAxSFVVZERhWE5IUVZGUlFncG5OemgzUVZKVlJWWm5lRlZoU0ZJd1kwaE5Oa3g1T1c1aFdGSnZaRmRKZFZreU9YUk1NbWd4V2pKa2NHSnRaRzFaVjA1c1RESjBiR050Tld4aVNFMTBDbGt5T1hSaVdGWjFZVmhTTlV3eVJtcGtSMngyWW01TmRtTnVWblZqZVRoNlQxUkplazlVVlhoUFJFVjNUa001YUdSSVVteGlXRUl3WTNrNGVFMUNXVWNLUTJselIwRlJVVUpuTnpoM1FWSlpSVU5CZDBkalNGWnBZa2RzYWsxRldVZERhWE5IUVZGUlFtYzNPSGRCVW1kRlQwRjNNbU50Vm5kaWVuQnZaRmRrYmdwaFZ6VnVXbTFHYWxwVE9YSmFXRXAxV2xkNGVreFhUblppVnpFeFltMXNNR1ZVY0hsYVYxazJZMjFXYldONU9XOWFWMFpyWTNrNWRGbFhiSFZOU1VkTENrSm5iM0pDWjBWRlFXUmFOVUZuVVVOQ1NIZEZaV2RDTkVGSVdVRXpWREIzWVhOaVNFVlVTbXBIVWpSamJWZGpNMEZ4U2t0WWNtcGxVRXN6TDJnMGNIa0taME00Y0Rkdk5FRkJRVWRvVm10d1NqQjNRVUZDUVUxQlVucENSa0ZwUW1RNVdUZHJNMDloYTJORVdFOVZUM1JRZVhacWRsUklXbUZWZWk5Q0wzUkdOQXBOTDJ4eVFWQmlVMnRuU1doQlRWTTJSRlJYTUdzMmIwdEpSV0V4VTBGVVdVaEhWMjlVYlZoaFJqTjBObXhGYTNSNGFreHlhalZQVFUxQmIwZERRM0ZIQ2xOTk5EbENRVTFFUVRKblFVMUhWVU5OU0RJelVFcG1jbmxIVFZwSWJUWTVkbU5oTjBzck5FTmxVMU5rZEVselVGSnZhR3hpU1RZeVExWlVVV2Q0THpVS2VrVk1SSEExTkZWSGVUSnpLM000VHpCQlNYaEJTVVJ1Y201WmFVNDBRMkZQZFhGU2JIQTJWM0JRVEZGMVJ6QllkbnByTTFoTkwybHZZVUppUTFZNWJRcEhaa2d4ZVRZM05DOXJja2ROWldaamJHcHBZMVYzUFQwS0xTMHRMUzFGVGtRZ1EwVlNWRWxHU1VOQlZFVXRMUzB0TFFvPSJ9fX19"}], "timestampVerificationData":{"rfc3161Timestamps":[{"signedTimestamp":"MIICyzADAgEAMIICwgYJKoZIhvcNAQcCoIICszCCAq8CAQMxDTALBglghkgBZQMEAgEwgbgGCyqGSIb3DQEJEAEEoIGoBIGlMIGiAgEBBgkrBgEEAYO/MAIwMTANBglghkgBZQMEAgEFAAQghhLtVKNwP9C9AoYe4HEUs+7awj5hU/huvyNh5KwKZwoCFQDsr/tPNk6bbSMFPHHhg8PZMEWfhxgPMjAyNjEwMTkyMjMxMTFaMAMCAQGgMqQwMC4xFTATBgNVBAoTDHNpZ3N0b3JlLmRldjEVMBMGA1UEAxMMc2lnc3RvcmUtdHNhoAAxggHcMIIB2AIBATBRMDkxFTATBgNVBAoTDHNpZ3N0b3JlLmRldjEgMB4GA1UEAxMXc2lnc3RvcmUtdHNhLXNlbGZzaWduZWQCFDoTVC8MkGHuvMFDL8uKjosqI4sMMAsGCWCGSAFlAwQCAaCB/DAaBgkqhkiG9w0BCQMxDQYLKoZIhvcNAQkQAQQwHAYJKoZIhvcNAQkFMQ8XDTI2MTAxOTIyMzExMVowLwYJKoZIhvcNAQkEMSIEIAVFYYjePZj3iCjN5TwF261bkicR+j86cqwhc/Ylrv1OMIGOBgsqhkiG9w0BCRACLzF/MH0wezB5BCCF+Se8B6tiysO0Q1bBDvyBssaIP9p6uebYcNnROs0FtzBVMD2kOzA5MRUwEwYDVQQKEwxzaWdzdG9yZS5kZXYxIDAeBgNVBAMTF3NpZ3N0b3JlLXRzYS1zZWxmc2lnbmVkAhQ6E1QvDJBh7rzBQy/Lio6LKiOLDDAKBggqhkjOPQQDAgRoMGYCMQCaOXxeRZ9FR864IHhKG6YZGWsx5NQvJZPbeYBSvP+tMDPnsAAdW0y0QUPJTGc2qLQCMQCaRwSvFIurXXHVpNG8n/+B9jzewECiPpGV67deqVZWywtL/TmaVHNu8YpEgd9efms="}]}}, "messageSignature":{"messageDigest":{"algorithm":"SHA2_256", "digest":"kfjUF8vBIGiyINBvj/FYkwtXznX1w5OXkUCYB0Wo9UI="}, "signature":"MEQCIGHjXRnJjTnqZiVuBjMeOhaNxhr0HQ6zzRrB4qPe7V93AiBoPy/dOQKqVM9igLjRnhNxQACZe/38afKHPj0LMfNMuA=="}}
build/torch210-rocm71-x86_64-linux/mha.py ADDED
@@ -0,0 +1,332 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FlashAttention forward entry points backed by aiter's Composable-Kernel FMHA.
2
+
3
+ This is the compiled (HIP / Composable Kernel) counterpart to the Triton
4
+ ``aiter-flash-attn`` package. Only the *forward* pass is provided (dense and
5
+ variable-length), including support for learnable attention sinks via
6
+ ``window_size[2]`` / ``s_aux`` (e.g. gpt-oss). The op set ships exclusively
7
+ the CK ``mha_fwd`` / ``mha_varlen_fwd`` kernels, so the sink path is always
8
+ taken (no fall-through to a non-sink ASM kernel).
9
+ """
10
+
11
+ from typing import Optional, Tuple, Union
12
+
13
+ import torch
14
+ from torch import Tensor
15
+
16
+ from ._ops import ops
17
+
18
+
19
+ def _round_multiple(x: int, m: int) -> int:
20
+ return (x + m - 1) // m * m
21
+
22
+
23
+ def _maybe_pad_headdim(t: Tensor, target: int) -> Tensor:
24
+ og = t.size(-1)
25
+ if og == target:
26
+ return t
27
+ return torch.nn.functional.pad(t, [0, target - og])
28
+
29
+
30
+ def _unpack_window(window_size) -> Tuple[int, int, int]:
31
+ left = int(window_size[0])
32
+ right = int(window_size[1])
33
+ sink = int(window_size[2]) if len(window_size) == 3 else 0
34
+ return left, right, sink
35
+
36
+
37
+ def _prep_sink(s_aux: Optional[Tensor], nheads: int, device) -> Optional[Tensor]:
38
+ if s_aux is None:
39
+ return None
40
+ assert s_aux.device == device, "s_aux must be on the same device as q"
41
+ assert s_aux.shape[0] == nheads, "s_aux must have shape [nheads_q]"
42
+ if s_aux.dtype != torch.float32:
43
+ s_aux = s_aux.to(torch.float32)
44
+ return s_aux
45
+
46
+
47
+ def _effective_sink_size(sink_size: int, s_aux: Optional[Tensor]) -> int:
48
+ # The CK kernel only takes the sink path when sink_size > 0. Callers using
49
+ # the transformers attention interface pass the per-head sink logits as
50
+ # ``s_aux`` but leave ``window_size`` a 2-tuple (sink_size == 0), so enable
51
+ # a single learnable sink automatically when s_aux is given.
52
+ if s_aux is not None and sink_size == 0:
53
+ return 1
54
+ return sink_size
55
+
56
+
57
+ def flash_attn_func(
58
+ q: Tensor,
59
+ k: Tensor,
60
+ v: Tensor,
61
+ dropout_p: float = 0.0,
62
+ softmax_scale: Optional[float] = None,
63
+ causal: bool = False,
64
+ window_size: Tuple[int, ...] = (-1, -1, 0),
65
+ bias: Optional[Tensor] = None,
66
+ alibi_slopes: Optional[Tensor] = None,
67
+ return_lse: bool = False,
68
+ return_attn_probs: bool = False,
69
+ s_aux: Optional[Tensor] = None,
70
+ **kwargs,
71
+ ) -> Union[Tensor, Tuple[Tensor, ...]]:
72
+ """Dense FlashAttention forward.
73
+
74
+ ``q, k, v`` have shape ``(batch, seqlen, nheads, headdim)``. ``window_size``
75
+ is ``(left, right, sink_size)``; ``s_aux`` is an optional ``(nheads_q,)``
76
+ fp32 tensor of learnable sink logits. Extra keyword arguments (e.g. those
77
+ passed by the transformers flash-attention interface) are ignored.
78
+ """
79
+ if softmax_scale is None:
80
+ softmax_scale = q.shape[-1] ** (-0.5)
81
+ window_size_left, window_size_right, sink_size = _unpack_window(window_size)
82
+
83
+ nheads_q = q.size(2)
84
+ s_aux = _prep_sink(s_aux, nheads_q, q.device)
85
+ sink_size = _effective_sink_size(sink_size, s_aux)
86
+
87
+ head_q_og = q.size(-1)
88
+ head_v_og = v.size(-1)
89
+ head_q = _round_multiple(head_q_og, 8)
90
+ head_v = _round_multiple(head_v_og, 8)
91
+ q = _maybe_pad_headdim(q, head_q)
92
+ k = _maybe_pad_headdim(k, head_q)
93
+ v = _maybe_pad_headdim(v, head_v)
94
+
95
+ return_softmax = return_attn_probs and dropout_p > 0
96
+ out, softmax_lse, S_dmask, _ = ops.mha_fwd(
97
+ q,
98
+ k,
99
+ v,
100
+ dropout_p,
101
+ softmax_scale,
102
+ causal,
103
+ window_size_left,
104
+ window_size_right,
105
+ sink_size,
106
+ return_lse or return_attn_probs,
107
+ return_softmax,
108
+ None, # cu_seqlens_q
109
+ None, # cu_seqlens_kv
110
+ None, # out
111
+ bias,
112
+ alibi_slopes,
113
+ None, # q_descale
114
+ None, # k_descale
115
+ None, # v_descale
116
+ s_aux,
117
+ None, # gen
118
+ )
119
+
120
+ out = out[..., :head_v_og]
121
+ if not (return_lse or return_attn_probs):
122
+ return out
123
+ result = (out,)
124
+ if return_lse:
125
+ result = result + (softmax_lse,)
126
+ if return_attn_probs:
127
+ result = result + (S_dmask,)
128
+ return result
129
+
130
+
131
+ def flash_attn_varlen_func(
132
+ q: Tensor,
133
+ k: Tensor,
134
+ v: Tensor,
135
+ cu_seqlens_q: Tensor,
136
+ cu_seqlens_k: Tensor,
137
+ max_seqlen_q: int,
138
+ max_seqlen_k: int,
139
+ dropout_p: float = 0.0,
140
+ softmax_scale: Optional[float] = None,
141
+ causal: bool = False,
142
+ window_size: Tuple[int, ...] = (-1, -1, 0),
143
+ bias: Optional[Tensor] = None,
144
+ alibi_slopes: Optional[Tensor] = None,
145
+ return_lse: bool = False,
146
+ return_attn_probs: bool = False,
147
+ block_table: Optional[Tensor] = None,
148
+ min_seqlen_q: int = 0,
149
+ logits_soft_cap: float = 0.0,
150
+ zero_tensors: bool = False,
151
+ cu_seqlens_q_padded: Optional[Tensor] = None,
152
+ cu_seqlens_k_padded: Optional[Tensor] = None,
153
+ s_aux: Optional[Tensor] = None,
154
+ **kwargs,
155
+ ) -> Union[Tensor, Tuple[Tensor, ...]]:
156
+ """Variable-length / packed FlashAttention forward.
157
+
158
+ ``q`` has shape ``(total_q, nheads, headdim)``; ``cu_seqlens_*`` are int32
159
+ cumulative offsets of shape ``(batch + 1,)``. Sinks are supported through
160
+ ``window_size[2]`` / ``s_aux`` as in :func:`flash_attn_func`. Extra keyword
161
+ arguments (e.g. from the transformers flash-attention interface) are ignored.
162
+ """
163
+ if softmax_scale is None:
164
+ softmax_scale = q.shape[-1] ** (-0.5)
165
+ window_size_left, window_size_right, sink_size = _unpack_window(window_size)
166
+
167
+ nheads_q = q.size(1)
168
+ s_aux = _prep_sink(s_aux, nheads_q, q.device)
169
+ sink_size = _effective_sink_size(sink_size, s_aux)
170
+
171
+ head_q_og = q.size(-1)
172
+ head_v_og = v.size(-1)
173
+ head_q = _round_multiple(head_q_og, 8)
174
+ head_v = _round_multiple(head_v_og, 8)
175
+ q = _maybe_pad_headdim(q, head_q)
176
+ k = _maybe_pad_headdim(k, head_q)
177
+ v = _maybe_pad_headdim(v, head_v)
178
+
179
+ return_softmax = return_attn_probs and dropout_p > 0
180
+ out, softmax_lse, S_dmask, _ = ops.mha_varlen_fwd(
181
+ q,
182
+ k,
183
+ v,
184
+ cu_seqlens_q,
185
+ cu_seqlens_k,
186
+ max_seqlen_q,
187
+ max_seqlen_k,
188
+ min_seqlen_q,
189
+ dropout_p,
190
+ softmax_scale,
191
+ logits_soft_cap,
192
+ zero_tensors,
193
+ causal,
194
+ window_size_left,
195
+ window_size_right,
196
+ sink_size,
197
+ return_lse or return_attn_probs,
198
+ return_softmax,
199
+ None, # out
200
+ block_table,
201
+ bias,
202
+ alibi_slopes,
203
+ None, # q_descale
204
+ None, # k_descale
205
+ None, # v_descale
206
+ None, # gen
207
+ cu_seqlens_q_padded,
208
+ cu_seqlens_k_padded,
209
+ s_aux,
210
+ )
211
+
212
+ out = out[..., :head_v_og]
213
+ if not (return_lse or return_attn_probs):
214
+ return out
215
+ result = (out,)
216
+ if return_lse:
217
+ result = result + (softmax_lse,)
218
+ if return_attn_probs:
219
+ result = result + (S_dmask,)
220
+ return result
221
+
222
+
223
+ def _append_to_contiguous_cache(k_cache, v_cache, k, v, cache_seqlens, Sq):
224
+ """In-place append of new k/v into a contiguous cache at per-batch offsets."""
225
+ B = k_cache.shape[0]
226
+ if isinstance(cache_seqlens, int):
227
+ cache_seqlens = torch.full((B,), cache_seqlens, device=k_cache.device, dtype=torch.long)
228
+ pos = cache_seqlens.view(B, 1).to(torch.long) + torch.arange(Sq, device=k_cache.device).view(1, Sq)
229
+ bidx = torch.arange(B, device=k_cache.device).view(B, 1).expand(B, Sq)
230
+ k_cache[bidx, pos] = k.to(k_cache.dtype)
231
+ v_cache[bidx, pos] = v.to(v_cache.dtype)
232
+
233
+
234
+ def flash_attn_with_kvcache(
235
+ q: Tensor,
236
+ k_cache: Tensor,
237
+ v_cache: Tensor,
238
+ k: Optional[Tensor] = None,
239
+ v: Optional[Tensor] = None,
240
+ rotary_cos: Optional[Tensor] = None,
241
+ rotary_sin: Optional[Tensor] = None,
242
+ cache_seqlens=None,
243
+ block_table: Optional[Tensor] = None,
244
+ softmax_scale: Optional[float] = None,
245
+ causal: bool = True,
246
+ window_size: Tuple[int, ...] = (-1, -1, 0),
247
+ s_aux: Optional[Tensor] = None,
248
+ return_lse: bool = False,
249
+ **kwargs,
250
+ ) -> Union[Tensor, Tuple[Tensor, ...]]:
251
+ """FlashAttention against a KV cache, built on the CK split-KV kernel.
252
+
253
+ ``q`` is ``(batch, seqlen_q, nheads, headdim)``. For a contiguous cache,
254
+ ``k_cache``/``v_cache`` are ``(batch, seqlen_cache, nheads_k, headdim)`` and
255
+ ``cache_seqlens`` (int or ``(batch,)``) is the number of valid cache entries
256
+ *before* appending. If ``k``/``v`` (the new tokens) are given, they are
257
+ written into the cache in place. The query attends to the valid prefix
258
+ (``cache_seqlens`` + ``seqlen_q``) with bottom-right causal alignment.
259
+
260
+ Notes / current limitations:
261
+ - Rotary embedding is not applied here; pass ``q``/``k`` already rotated
262
+ (``rotary_cos``/``rotary_sin`` are rejected).
263
+ - Paged caches are addressed via ``block_table`` (forwarded to the split-KV
264
+ varlen kernel); in-cache append for paged layouts must be done by the
265
+ caller (pass ``k=v=None``).
266
+ """
267
+ if rotary_cos is not None or rotary_sin is not None:
268
+ raise NotImplementedError(
269
+ "flash_attn_with_kvcache does not apply rotary; rotate q/k beforehand."
270
+ )
271
+ if softmax_scale is None:
272
+ softmax_scale = q.shape[-1] ** (-0.5)
273
+ window_size_left, window_size_right, sink_size = _unpack_window(window_size)
274
+ B, Sq, Hq, D = q.shape
275
+ s_aux = _prep_sink(s_aux, Hq, q.device)
276
+ sink_size = _effective_sink_size(sink_size, s_aux)
277
+
278
+ paged = block_table is not None
279
+
280
+ # ---- append new tokens into the cache (contiguous layout only) ----
281
+ if k is not None and not paged:
282
+ assert cache_seqlens is not None, "cache_seqlens required to append into the cache"
283
+ _append_to_contiguous_cache(k_cache, v_cache, k, v, cache_seqlens, Sq)
284
+ elif k is not None and paged:
285
+ raise NotImplementedError(
286
+ "in-cache append for paged layouts is not supported; append externally "
287
+ "and call with k=v=None."
288
+ )
289
+
290
+ appended = 0 if k is None else Sq
291
+
292
+ # ---- key lengths ----
293
+ if cache_seqlens is None:
294
+ klens = torch.full((B,), k_cache.shape[1], device=q.device, dtype=torch.long)
295
+ elif isinstance(cache_seqlens, int):
296
+ klens = torch.full((B,), cache_seqlens + appended, device=q.device, dtype=torch.long)
297
+ else:
298
+ klens = cache_seqlens.to(q.device, torch.long) + appended
299
+
300
+ uniform = bool((klens == klens[0]).all().item())
301
+
302
+ # ---- fast path: contiguous cache, uniform length -> dense kernel ----
303
+ if not paged and uniform:
304
+ L = int(klens[0].item())
305
+ return flash_attn_func(
306
+ q, k_cache[:, :L], v_cache[:, :L],
307
+ softmax_scale=softmax_scale, causal=causal,
308
+ window_size=(window_size_left, window_size_right, sink_size),
309
+ s_aux=s_aux, return_lse=return_lse,
310
+ )
311
+
312
+ # ---- general path: variable cache lengths -> split-KV varlen kernel ----
313
+ cu_q = torch.arange(0, (B + 1) * Sq, Sq, device=q.device, dtype=torch.int32)
314
+ cu_k = torch.zeros(B + 1, device=q.device, dtype=torch.int32)
315
+ cu_k[1:] = torch.cumsum(klens, 0).to(torch.int32)
316
+ q_packed = q.reshape(B * Sq, Hq, D)
317
+ if paged:
318
+ k_packed, v_packed = k_cache, v_cache # kernel gathers via block_table
319
+ else:
320
+ k_packed = torch.cat([k_cache[b, : int(klens[b])] for b in range(B)], dim=0)
321
+ v_packed = torch.cat([v_cache[b, : int(klens[b])] for b in range(B)], dim=0)
322
+
323
+ out = flash_attn_varlen_func(
324
+ q_packed, k_packed, v_packed, cu_q, cu_k, Sq, int(klens.max().item()),
325
+ softmax_scale=softmax_scale, causal=causal,
326
+ window_size=(window_size_left, window_size_right, sink_size),
327
+ block_table=block_table, s_aux=s_aux, return_lse=return_lse,
328
+ )
329
+ if return_lse:
330
+ out, lse = out
331
+ return out.reshape(B, Sq, Hq, D), lse
332
+ return out.reshape(B, Sq, Hq, D)
build/torch210-rocm71-x86_64-linux/symbols.json ADDED
@@ -0,0 +1,349 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "schema_version": 1,
3
+ "module": "_aiter_flash_attn_ck_rocm_fe2bb66",
4
+ "functions": [
5
+ {
6
+ "name": "flash_attn_func",
7
+ "qualname": "flash_attn_func",
8
+ "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha",
9
+ "kind": "function",
10
+ "doc": "Dense FlashAttention forward.\n\n``q, k, v`` have shape ``(batch, seqlen, nheads, headdim)``. ``window_size``\nis ``(left, right, sink_size)``; ``s_aux`` is an optional ``(nheads_q,)``\nfp32 tensor of learnable sink logits. Extra keyword arguments (e.g. those\npassed by the transformers flash-attention interface) are ignored.",
11
+ "signature": {
12
+ "parameters": [
13
+ {
14
+ "name": "q",
15
+ "kind": "POSITIONAL_OR_KEYWORD",
16
+ "default": null,
17
+ "annotation": "torch.Tensor"
18
+ },
19
+ {
20
+ "name": "k",
21
+ "kind": "POSITIONAL_OR_KEYWORD",
22
+ "default": null,
23
+ "annotation": "torch.Tensor"
24
+ },
25
+ {
26
+ "name": "v",
27
+ "kind": "POSITIONAL_OR_KEYWORD",
28
+ "default": null,
29
+ "annotation": "torch.Tensor"
30
+ },
31
+ {
32
+ "name": "dropout_p",
33
+ "kind": "POSITIONAL_OR_KEYWORD",
34
+ "default": "0.0",
35
+ "annotation": "float"
36
+ },
37
+ {
38
+ "name": "softmax_scale",
39
+ "kind": "POSITIONAL_OR_KEYWORD",
40
+ "default": "None",
41
+ "annotation": "float | None"
42
+ },
43
+ {
44
+ "name": "causal",
45
+ "kind": "POSITIONAL_OR_KEYWORD",
46
+ "default": "False",
47
+ "annotation": "bool"
48
+ },
49
+ {
50
+ "name": "window_size",
51
+ "kind": "POSITIONAL_OR_KEYWORD",
52
+ "default": "(-1, -1, 0)",
53
+ "annotation": "Tuple[int, ...]"
54
+ },
55
+ {
56
+ "name": "bias",
57
+ "kind": "POSITIONAL_OR_KEYWORD",
58
+ "default": "None",
59
+ "annotation": "torch.Tensor | None"
60
+ },
61
+ {
62
+ "name": "alibi_slopes",
63
+ "kind": "POSITIONAL_OR_KEYWORD",
64
+ "default": "None",
65
+ "annotation": "torch.Tensor | None"
66
+ },
67
+ {
68
+ "name": "return_lse",
69
+ "kind": "POSITIONAL_OR_KEYWORD",
70
+ "default": "False",
71
+ "annotation": "bool"
72
+ },
73
+ {
74
+ "name": "return_attn_probs",
75
+ "kind": "POSITIONAL_OR_KEYWORD",
76
+ "default": "False",
77
+ "annotation": "bool"
78
+ },
79
+ {
80
+ "name": "s_aux",
81
+ "kind": "POSITIONAL_OR_KEYWORD",
82
+ "default": "None",
83
+ "annotation": "torch.Tensor | None"
84
+ },
85
+ {
86
+ "name": "kwargs",
87
+ "kind": "VAR_KEYWORD",
88
+ "default": null,
89
+ "annotation": null
90
+ }
91
+ ],
92
+ "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]"
93
+ }
94
+ },
95
+ {
96
+ "name": "flash_attn_varlen_func",
97
+ "qualname": "flash_attn_varlen_func",
98
+ "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha",
99
+ "kind": "function",
100
+ "doc": "Variable-length / packed FlashAttention forward.\n\n``q`` has shape ``(total_q, nheads, headdim)``; ``cu_seqlens_*`` are int32\ncumulative offsets of shape ``(batch + 1,)``. Sinks are supported through\n``window_size[2]`` / ``s_aux`` as in :func:`flash_attn_func`. Extra keyword\narguments (e.g. from the transformers flash-attention interface) are ignored.",
101
+ "signature": {
102
+ "parameters": [
103
+ {
104
+ "name": "q",
105
+ "kind": "POSITIONAL_OR_KEYWORD",
106
+ "default": null,
107
+ "annotation": "torch.Tensor"
108
+ },
109
+ {
110
+ "name": "k",
111
+ "kind": "POSITIONAL_OR_KEYWORD",
112
+ "default": null,
113
+ "annotation": "torch.Tensor"
114
+ },
115
+ {
116
+ "name": "v",
117
+ "kind": "POSITIONAL_OR_KEYWORD",
118
+ "default": null,
119
+ "annotation": "torch.Tensor"
120
+ },
121
+ {
122
+ "name": "cu_seqlens_q",
123
+ "kind": "POSITIONAL_OR_KEYWORD",
124
+ "default": null,
125
+ "annotation": "torch.Tensor"
126
+ },
127
+ {
128
+ "name": "cu_seqlens_k",
129
+ "kind": "POSITIONAL_OR_KEYWORD",
130
+ "default": null,
131
+ "annotation": "torch.Tensor"
132
+ },
133
+ {
134
+ "name": "max_seqlen_q",
135
+ "kind": "POSITIONAL_OR_KEYWORD",
136
+ "default": null,
137
+ "annotation": "int"
138
+ },
139
+ {
140
+ "name": "max_seqlen_k",
141
+ "kind": "POSITIONAL_OR_KEYWORD",
142
+ "default": null,
143
+ "annotation": "int"
144
+ },
145
+ {
146
+ "name": "dropout_p",
147
+ "kind": "POSITIONAL_OR_KEYWORD",
148
+ "default": "0.0",
149
+ "annotation": "float"
150
+ },
151
+ {
152
+ "name": "softmax_scale",
153
+ "kind": "POSITIONAL_OR_KEYWORD",
154
+ "default": "None",
155
+ "annotation": "float | None"
156
+ },
157
+ {
158
+ "name": "causal",
159
+ "kind": "POSITIONAL_OR_KEYWORD",
160
+ "default": "False",
161
+ "annotation": "bool"
162
+ },
163
+ {
164
+ "name": "window_size",
165
+ "kind": "POSITIONAL_OR_KEYWORD",
166
+ "default": "(-1, -1, 0)",
167
+ "annotation": "Tuple[int, ...]"
168
+ },
169
+ {
170
+ "name": "bias",
171
+ "kind": "POSITIONAL_OR_KEYWORD",
172
+ "default": "None",
173
+ "annotation": "torch.Tensor | None"
174
+ },
175
+ {
176
+ "name": "alibi_slopes",
177
+ "kind": "POSITIONAL_OR_KEYWORD",
178
+ "default": "None",
179
+ "annotation": "torch.Tensor | None"
180
+ },
181
+ {
182
+ "name": "return_lse",
183
+ "kind": "POSITIONAL_OR_KEYWORD",
184
+ "default": "False",
185
+ "annotation": "bool"
186
+ },
187
+ {
188
+ "name": "return_attn_probs",
189
+ "kind": "POSITIONAL_OR_KEYWORD",
190
+ "default": "False",
191
+ "annotation": "bool"
192
+ },
193
+ {
194
+ "name": "block_table",
195
+ "kind": "POSITIONAL_OR_KEYWORD",
196
+ "default": "None",
197
+ "annotation": "torch.Tensor | None"
198
+ },
199
+ {
200
+ "name": "min_seqlen_q",
201
+ "kind": "POSITIONAL_OR_KEYWORD",
202
+ "default": "0",
203
+ "annotation": "int"
204
+ },
205
+ {
206
+ "name": "logits_soft_cap",
207
+ "kind": "POSITIONAL_OR_KEYWORD",
208
+ "default": "0.0",
209
+ "annotation": "float"
210
+ },
211
+ {
212
+ "name": "zero_tensors",
213
+ "kind": "POSITIONAL_OR_KEYWORD",
214
+ "default": "False",
215
+ "annotation": "bool"
216
+ },
217
+ {
218
+ "name": "cu_seqlens_q_padded",
219
+ "kind": "POSITIONAL_OR_KEYWORD",
220
+ "default": "None",
221
+ "annotation": "torch.Tensor | None"
222
+ },
223
+ {
224
+ "name": "cu_seqlens_k_padded",
225
+ "kind": "POSITIONAL_OR_KEYWORD",
226
+ "default": "None",
227
+ "annotation": "torch.Tensor | None"
228
+ },
229
+ {
230
+ "name": "s_aux",
231
+ "kind": "POSITIONAL_OR_KEYWORD",
232
+ "default": "None",
233
+ "annotation": "torch.Tensor | None"
234
+ },
235
+ {
236
+ "name": "kwargs",
237
+ "kind": "VAR_KEYWORD",
238
+ "default": null,
239
+ "annotation": null
240
+ }
241
+ ],
242
+ "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]"
243
+ }
244
+ },
245
+ {
246
+ "name": "flash_attn_with_kvcache",
247
+ "qualname": "flash_attn_with_kvcache",
248
+ "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha",
249
+ "kind": "function",
250
+ "doc": "FlashAttention against a KV cache, built on the CK split-KV kernel.\n\n``q`` is ``(batch, seqlen_q, nheads, headdim)``. For a contiguous cache,\n``k_cache``/``v_cache`` are ``(batch, seqlen_cache, nheads_k, headdim)`` and\n``cache_seqlens`` (int or ``(batch,)``) is the number of valid cache entries\n*before* appending. If ``k``/``v`` (the new tokens) are given, they are\nwritten into the cache in place. The query attends to the valid prefix\n(``cache_seqlens`` + ``seqlen_q``) with bottom-right causal alignment.\n\nNotes / current limitations:\n- Rotary embedding is not applied here; pass ``q``/``k`` already rotated\n (``rotary_cos``/``rotary_sin`` are rejected).\n- Paged caches are addressed via ``block_table`` (forwarded to the split-KV\n varlen kernel); in-cache append for paged layouts must be done by the\n caller (pass ``k=v=None``).",
251
+ "signature": {
252
+ "parameters": [
253
+ {
254
+ "name": "q",
255
+ "kind": "POSITIONAL_OR_KEYWORD",
256
+ "default": null,
257
+ "annotation": "torch.Tensor"
258
+ },
259
+ {
260
+ "name": "k_cache",
261
+ "kind": "POSITIONAL_OR_KEYWORD",
262
+ "default": null,
263
+ "annotation": "torch.Tensor"
264
+ },
265
+ {
266
+ "name": "v_cache",
267
+ "kind": "POSITIONAL_OR_KEYWORD",
268
+ "default": null,
269
+ "annotation": "torch.Tensor"
270
+ },
271
+ {
272
+ "name": "k",
273
+ "kind": "POSITIONAL_OR_KEYWORD",
274
+ "default": "None",
275
+ "annotation": "torch.Tensor | None"
276
+ },
277
+ {
278
+ "name": "v",
279
+ "kind": "POSITIONAL_OR_KEYWORD",
280
+ "default": "None",
281
+ "annotation": "torch.Tensor | None"
282
+ },
283
+ {
284
+ "name": "rotary_cos",
285
+ "kind": "POSITIONAL_OR_KEYWORD",
286
+ "default": "None",
287
+ "annotation": "torch.Tensor | None"
288
+ },
289
+ {
290
+ "name": "rotary_sin",
291
+ "kind": "POSITIONAL_OR_KEYWORD",
292
+ "default": "None",
293
+ "annotation": "torch.Tensor | None"
294
+ },
295
+ {
296
+ "name": "cache_seqlens",
297
+ "kind": "POSITIONAL_OR_KEYWORD",
298
+ "default": "None",
299
+ "annotation": null
300
+ },
301
+ {
302
+ "name": "block_table",
303
+ "kind": "POSITIONAL_OR_KEYWORD",
304
+ "default": "None",
305
+ "annotation": "torch.Tensor | None"
306
+ },
307
+ {
308
+ "name": "softmax_scale",
309
+ "kind": "POSITIONAL_OR_KEYWORD",
310
+ "default": "None",
311
+ "annotation": "float | None"
312
+ },
313
+ {
314
+ "name": "causal",
315
+ "kind": "POSITIONAL_OR_KEYWORD",
316
+ "default": "True",
317
+ "annotation": "bool"
318
+ },
319
+ {
320
+ "name": "window_size",
321
+ "kind": "POSITIONAL_OR_KEYWORD",
322
+ "default": "(-1, -1, 0)",
323
+ "annotation": "Tuple[int, ...]"
324
+ },
325
+ {
326
+ "name": "s_aux",
327
+ "kind": "POSITIONAL_OR_KEYWORD",
328
+ "default": "None",
329
+ "annotation": "torch.Tensor | None"
330
+ },
331
+ {
332
+ "name": "return_lse",
333
+ "kind": "POSITIONAL_OR_KEYWORD",
334
+ "default": "False",
335
+ "annotation": "bool"
336
+ },
337
+ {
338
+ "name": "kwargs",
339
+ "kind": "VAR_KEYWORD",
340
+ "default": null,
341
+ "annotation": null
342
+ }
343
+ ],
344
+ "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]"
345
+ }
346
+ }
347
+ ],
348
+ "layers": []
349
+ }
build/torch211-rocm71-x86_64-linux/__init__.py ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """AITER Composable-Kernel FlashAttention kernels for AMD ROCm (compiled HIP).
2
+
3
+ Compiled (Composable Kernel / HIP) counterpart to the Triton-based
4
+ ``aiter-flash-attn`` package, repackaged from the CK FMHA forward path of the
5
+ `ROCm/aiter <https://github.com/ROCm/aiter>`_ project as a self-contained
6
+ Hugging Face Hub kernel. Exposes the FlashAttention forward entry points with
7
+ learnable attention sink support (e.g. gpt-oss).
8
+ """
9
+
10
+ from .mha import (
11
+ flash_attn_func,
12
+ flash_attn_varlen_func,
13
+ flash_attn_with_kvcache,
14
+ )
15
+
16
+
17
+ __kernel_metadata__ = {
18
+ "license": "mit",
19
+ }
20
+
21
+
22
+ __all__ = [
23
+ "__kernel_metadata__",
24
+ "flash_attn_func",
25
+ "flash_attn_varlen_func",
26
+ "flash_attn_with_kvcache",
27
+ ]
build/torch211-rocm71-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8a386f0a7da580b6efa0dd9ffac2d1a4be1382a709f490ab786bf184b7fb4744
3
+ size 204251704
build/torch211-rocm71-x86_64-linux/_ops.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _aiter_flash_attn_ck_rocm_fe2bb66
3
+ ops = torch.ops._aiter_flash_attn_ck_rocm_fe2bb66
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ """
7
+ Prefix op by namespace.
8
+ """
9
+ return f"_aiter_flash_attn_ck_rocm_fe2bb66::{op_name}"
build/torch211-rocm71-x86_64-linux/metadata.json ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "aiter-flash-attn-ck",
3
+ "id": "_aiter_flash_attn_ck_rocm_fe2bb66",
4
+ "version": 1,
5
+ "kernels-minver": "0.14.0",
6
+ "license": "MIT",
7
+ "python-depends": [],
8
+ "kernel-depends": [],
9
+ "backend": {
10
+ "type": "rocm",
11
+ "archs": [
12
+ "gfx942",
13
+ "gfx950"
14
+ ]
15
+ },
16
+ "digest": {
17
+ "algorithm": "sha256",
18
+ "files": {
19
+ "__init__.py": "7wj1GTm2bqTLLCyT4xLuAk/+JWYrhINjY/ODMNWhgZ0=",
20
+ "_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so": "ijhvCn2lgLbvoN2f+sLRpL4TgqcJ9JCreGvxhLf7R0Q=",
21
+ "_ops.py": "4WkdYDWWJp60zep+oW/5ym9vIeVMNfrgRCkP3o/w7yU=",
22
+ "mha.py": "zCAjY4O+DiCzmEL+W9sqXxZaqjsj4RJbWFcyPpUZejk=",
23
+ "symbols.json": "rhnF4H8Jsb6I+YPxIVFMgnLcTQJODEXbZ7ge77NZXAU="
24
+ }
25
+ },
26
+ "provenance": {
27
+ "kernel-builder": {
28
+ "version": "0.18.0-dev0",
29
+ "commit": "d054906a13710b811b96697b4635f03020842733",
30
+ "dirty": false
31
+ },
32
+ "kernel": {
33
+ "commit": "fe2bb66fee6c2a77eccf08a0508dc57428d488ed",
34
+ "dirty": false
35
+ }
36
+ }
37
+ }
build/torch211-rocm71-x86_64-linux/metadata.json.sigstore ADDED
@@ -0,0 +1 @@
 
 
1
+ {"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json", "verificationMaterial":{"certificate":{"rawBytes":"MIIHLTCCBrKgAwIBAgIUEmpsGYHCGCL1f77EXMK1mpdslE0wCgYIKoZIzj0EAwMwNzEVMBMGA1UEChMMc2lnc3RvcmUuZGV2MR4wHAYDVQQDExVzaWdzdG9yZS1pbnRlcm1lZGlhdGUwHhcNMjYxMDE5MjIzMTEzWhcNMjYxMDE5MjI0MTEzWjAAMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE4Fy9kpEGzuCqkOEdVCP+RLOaIvA6RK5RuqsBRqXOQ1oE/UwcaBM8VLmKzmbspKP8LZxu8R+MYRHT7vmO2qJGLKOCBdEwggXNMA4GA1UdDwEB/wQEAwIHgDATBgNVHSUEDDAKBggrBgEFBQcDAzAfBgNVHSMEGDAWgBTf0+nPViQRlvmo2OkoVaLGLhhkPzBrBgNVHREBAf8EYTBfhl1odHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkvLmdpdGh1Yi93b3JrZmxvd3MvYnVpbGQueWFtbEByZWZzL2hlYWRzL21haW4wOQYKKwYBBAGDvzABAQQraHR0cHM6Ly90b2tlbi5hY3Rpb25zLmdpdGh1YnVzZXJjb250ZW50LmNvbTAfBgorBgEEAYO/MAECBBF3b3JrZmxvd19kaXNwYXRjaDA2BgorBgEEAYO/MAEDBChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMBMGCisGAQQBg78wAQQEBUJ1aWxkMCsGCisGAQQBg78wAQUEHWh1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5MB0GCisGAQQBg78wAQYED3JlZnMvaGVhZHMvbWFpbjA7BgorBgEEAYO/MAEIBC0MK2h0dHBzOi8vdG9rZW4uYWN0aW9ucy5naXRodWJ1c2VyY29udGVudC5jb20wbQYKKwYBBAGDvzABCQRfDF1odHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkvLmdpdGh1Yi93b3JrZmxvd3MvYnVpbGQueWFtbEByZWZzL2hlYWRzL21haW4wOAYKKwYBBAGDvzABCgQqDChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMBsGCisGAQQBg78wAQsEDQwLc2VsZi1ob3N0ZWQwQAYKKwYBBAGDvzABDAQyDDBodHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkwOAYKKwYBBAGDvzABDQQqDChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMB8GCisGAQQBg78wAQ4EEQwPcmVmcy9oZWFkcy9tYWluMBoGCisGAQQBg78wAQ8EDAwKMTA3MTQ3NTUyOTAuBgorBgEEAYO/MAEQBCAMHmh0dHBzOi8vZ2l0aHViLmNvbS9odWdnaW5nZmFjZTAYBgorBgEEAYO/MAERBAoMCDI1NzIwNzQzMG0GCisGAQQBg78wARIEXwxdaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5Ly5naXRodWIvd29ya2Zsb3dzL2J1aWxkLnlhbWxAcmVmcy9oZWFkcy9tYWluMDgGCisGAQQBg78wARMEKgwoZmUyYmI2NmZlZTZjMmE3N2VjY2YwOGEwNTA4ZGM1NzQyOGQ0ODhlZDAhBgorBgEEAYO/MAEUBBMMEXdvcmtmbG93X2Rpc3BhdGNoMGQGCisGAQQBg78wARUEVgxUaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5L2FjdGlvbnMvcnVucy8zOTIzOTUxODEwNC9hdHRlbXB0cy8xMBYGCisGAQQBg78wARYECAwGcHVibGljMEYGCisGAQQBg78wARgEOAw2cmVwbzpodWdnaW5nZmFjZS9rZXJuZWxzLWNvbW11bml0eTpyZWY6cmVmcy9oZWFkcy9tYWluMIGKBgorBgEEAdZ5AgQCBHwEegB4AHYA3T0wasbHETJjGR4cmWc3AqJKXrjePK3/h4pygC8p7o4AAAGhVkpQHwAABAMARzBFAiEA5+eh0gx7+UP+mTMZ7dYGtXAGoMr/u3B/1ATobugZirgCIDMCumLOPPvo9Ns0vC8ybrQt7fXH5h98acJfF9aYV1fuMAoGCCqGSM49BAMDA2kAMGYCMQC1gggEuYtV0xjSBQFLnBzb2NdSMXXnrwNQMbOsqeldlpTGoUouHMgT0NboOvp74NcCMQCy79wgE2YbxXDZW7ye5v9a8T72ozdyyAG3+bFixIHj0rx86inTdkuqjr8I90iw9ok="}, "tlogEntries":[{"logIndex":"3327211453", "logId":{"keyId":"wNI9atQGlz+VWfO6LRygH4QUfY/8W4RFwiT5i5WRgB0="}, "kindVersion":{"kind":"hashedrekord", "version":"0.0.1"}, "integratedTime":"1792449073", "inclusionPromise":{"signedEntryTimestamp":"MEQCICqb2FveJcObQ1buwFT+0NvR4ZCa0apWIyoOzyrWsgzYAiB4YVmCelHrY9h3rlc+qWSF+O+hNiQ/syPijjNsHJwahg=="}, "inclusionProof":{"logIndex":"3205307191", "rootHash":"iiOp2twenBLzbfls9348G+Zypx6NFcfs3+o2fG/C/fk=", "treeSize":"3205307194", "hashes":["fquilGZratvzTDiv84GzP8s55UYAN0iUI2kGqK20cXE=", "fugeqQEriw+je6JcoVNkTkv+UD+IzIQtDv/zaj8Quj8=", "4hRzU/5ELB96P1nZkhgwpYYVgFAr1DXDztt20pO7If4=", "rM02OjNrzut1+wWBYgEkyMNTWk3J8j72VYpb5yxWm/g=", "gIcTbwmbW4Jinimt4uWlbE/g8i6p+zi5IXTWev0jxFY=", "iekAzKnn2vj8KKbMbX8Obrd5/3Mxh3/6lsToOqKeuTU=", "AJlvaM7WQKZg0FAjSdh7RzJXyGKaOW5DVz46FbaUPZE=", "t4Yy4kaoiJ3+VosqXw5Ogmg+A7p/E6y8+JhqWbalMbI=", "5egUlZ4Pi0Ix/qLyCDXocsWywJCHqrde/S06vOZvlOM=", "QsXufUpoKtSjKAB19K1G00umQpWvPHlBNuRGFiiVjQQ=", "QIMkwQbwAewhLJd//5FFwI3SnyNQBGjCgJrelfG8vhc=", "Et17CACUv9QN64irUJuBjsrgRGkGnnJ9xacRV/aBdwQ=", "lqgEwkxhDMs/gdIAE4lmjfMwg7nDmv7lda6O0LGliCw=", "HovTPBWzoErvUg1u5sIi3x1ZonZ7yJekVsBrmpomCIA=", "G+nk7mgE5h6dUmv2UY7NDJrY3WvrZY31raaO1AETt+o=", "qlTWNDzERnTBQVcHHXXVPjiUP0SsikGxfcxVlgqVFDo=", "gg9GNApNke3+zNhl4tsWtmIycCFrOMCB3lys6XuSQNQ=", "NlOclldmKPA1tMKIskqHAwuQDYHp31YwpK+OOMrMccc=", "qxzHanAzz57SDdmJe0B7bJK72NTIBbwEMGBBKvDOROw=", "xH/DCseLHr9eKoYT8qsORZK7zVdEGYWHuVtsVrD95wY="], "checkpoint":{"envelope":"rekor.sigstore.dev - 1193050959916656506\n3205307194\niiOp2twenBLzbfls9348G+Zypx6NFcfs3+o2fG/C/fk=\n\n— rekor.sigstore.dev wNI9ajBGAiEA4MrXZ0/pNdLZFmSAyN8SsGiWgHq/FNXsTxStKfSyb5YCIQD4kmm+enQutJHTg7MkOS+ckvQ2VGJuDl8kjANqXlKi6Q==\n"}}, "canonicalizedBody":"eyJhcGlWZXJzaW9uIjoiMC4wLjEiLCJraW5kIjoiaGFzaGVkcmVrb3JkIiwic3BlYyI6eyJkYXRhIjp7Imhhc2giOnsiYWxnb3JpdGhtIjoic2hhMjU2IiwidmFsdWUiOiI1MGE4MDNkMjI4NDhkOGExYThmNmRhYWRmNWJkZTVhYWQ2ZDk3MzUwNjI1OTIwY2E4OTVmYzAyNDM2ZjVkNDQ1In19LCJzaWduYXR1cmUiOnsiY29udGVudCI6Ik1FUUNJQm8rZ3Ftb09qeUcxNVFpUkRRd3d4QTNuMVExVi83dUJrOFRLYlZ5ZEpkOEFpQnFNcC96SlZkditjN1IzSHM2UlFnbE5EQmFhZnJYNzRUZ25TQzJ0dDdTTEE9PSIsInB1YmxpY0tleSI6eyJjb250ZW50IjoiTFMwdExTMUNSVWRKVGlCRFJWSlVTVVpKUTBGVVJTMHRMUzB0Q2sxSlNVaE1WRU5EUW5KTFowRjNTVUpCWjBsVlJXMXdjMGRaU0VOSFEwd3haamMzUlZoTlN6RnRjR1J6YkVVd2QwTm5XVWxMYjFwSmVtb3dSVUYzVFhjS1RucEZWazFDVFVkQk1WVkZRMmhOVFdNeWJHNWpNMUoyWTIxVmRWcEhWakpOVWpSM1NFRlpSRlpSVVVSRmVGWjZZVmRrZW1SSE9YbGFVekZ3WW01U2JBcGpiVEZzV2tkc2FHUkhWWGRJYUdOT1RXcFplRTFFUlRWTmFrbDZUVlJGZWxkb1kwNU5hbGw0VFVSRk5VMXFTVEJOVkVWNlYycEJRVTFHYTNkRmQxbElDa3R2V2tsNmFqQkRRVkZaU1V0dldrbDZhakJFUVZGalJGRm5RVVUwUm5rNWEzQkZSM3AxUTNGclQwVmtWa05RSzFKTVQyRkpka0UyVWtzMVVuVnhjMElLVW5GWVQxRXhiMFV2VlhkallVSk5PRlpNYlV0NmJXSnpjRXRRT0V4YWVIVTRVaXROV1ZKSVZEZDJiVTh5Y1VwSFRFdFBRMEprUlhkbloxaE9UVUUwUndwQk1WVmtSSGRGUWk5M1VVVkJkMGxJWjBSQlZFSm5UbFpJVTFWRlJFUkJTMEpuWjNKQ1owVkdRbEZqUkVGNlFXWkNaMDVXU0ZOTlJVZEVRVmRuUWxSbUNqQXJibEJXYVZGU2JIWnRiekpQYTI5V1lVeEhUR2hvYTFCNlFuSkNaMDVXU0ZKRlFrRm1PRVZaVkVKbWFHd3hiMlJJVW5kamVtOTJUREprY0dSSGFERUtXV2sxYW1JeU1IWmhTRlp1V2pKc2RWb3lXbWhaTWxWMllUSldlV0p0Vm5OamVURnFZakl4ZEdSWE5YQmtTR3QyVEcxa2NHUkhhREZaYVRrellqTktjZ3BhYlhoMlpETk5kbGx1Vm5CaVIxRjFaVmRHZEdKRlFubGFWMXA2VERKb2JGbFhVbnBNTWpGb1lWYzBkMDlSV1V0TGQxbENRa0ZIUkhaNlFVSkJVVkZ5Q21GSVVqQmpTRTAyVEhrNU1HSXlkR3hpYVRWb1dUTlNjR0l5TlhwTWJXUndaRWRvTVZsdVZucGFXRXBxWWpJMU1GcFhOVEJNYlU1MllsUkJaa0puYjNJS1FtZEZSVUZaVHk5TlFVVkRRa0pHTTJJelNuSmFiWGgyWkRFNWEyRllUbmRaV0ZKcVlVUkJNa0puYjNKQ1owVkZRVmxQTDAxQlJVUkNRMmh0V2xSS2FRcFphbGt5V20xV2JFNXRUWGxaVkdNeldsZE9hbHBxUVRSWlZFRXhUVVJvYTFsNlZUTk9SRWswV2tSUk5FOUhWbXROUWsxSFEybHpSMEZSVVVKbk56aDNDa0ZSVVVWQ1ZVb3hZVmQ0YTAxRGMwZERhWE5IUVZGUlFtYzNPSGRCVVZWRlNGZG9NVm95WkhCaWJXUnRXVmRPYkV3eWRHeGpiVFZzWWtoTmRGa3lPWFFLWWxoV2RXRllValZOUWpCSFEybHpSMEZSVVVKbk56aDNRVkZaUlVRelNteGFiazEyWVVkV2FGcElUWFppVjBad1ltcEJOMEpuYjNKQ1owVkZRVmxQTHdwTlFVVkpRa013VFVzeWFEQmtTRUo2VDJrNGRtUkhPWEphVnpSMVdWZE9NR0ZYT1hWamVUVnVZVmhTYjJSWFNqRmpNbFo1V1RJNWRXUkhWblZrUXpWcUNtSXlNSGRpVVZsTFMzZFpRa0pCUjBSMmVrRkNRMUZTWmtSR01XOWtTRkozWTNwdmRrd3laSEJrUjJneFdXazFhbUl5TUhaaFNGWnVXakpzZFZveVdtZ0tXVEpWZG1FeVZubGliVlp6WTNreGFtSXlNWFJrVnpWd1pFaHJka3h0WkhCa1IyZ3hXV2s1TTJJelNuSmFiWGgyWkROTmRsbHVWbkJpUjFGMVpWZEdkQXBpUlVKNVdsZGFla3d5YUd4WlYxSjZUREl4YUdGWE5IZFBRVmxMUzNkWlFrSkJSMFIyZWtGQ1EyZFJjVVJEYUcxYVZFcHBXV3BaTWxwdFZteE9iVTE1Q2xsVVl6TmFWMDVxV21wQk5GbFVRVEZOUkdocldYcFZNMDVFU1RSYVJGRTBUMGRXYTAxQ2MwZERhWE5IUVZGUlFtYzNPSGRCVVhORlJGRjNUR015Vm5NS1dta3hiMkl6VGpCYVYxRjNVVUZaUzB0M1dVSkNRVWRFZG5wQlFrUkJVWGxFUkVKdlpFaFNkMk42YjNaTU1tUndaRWRvTVZscE5XcGlNakIyWVVoV2JncGFNbXgxV2pKYWFGa3lWWFpoTWxaNVltMVdjMk41TVdwaU1qRjBaRmMxY0dSSWEzZFBRVmxMUzNkWlFrSkJSMFIyZWtGQ1JGRlJjVVJEYUcxYVZFcHBDbGxxV1RKYWJWWnNUbTFOZVZsVVl6TmFWMDVxV21wQk5GbFVRVEZOUkdocldYcFZNMDVFU1RSYVJGRTBUMGRXYTAxQ09FZERhWE5IUVZGUlFtYzNPSGNLUVZFMFJVVlJkMUJqYlZadFkzazViMXBYUm10amVUbDBXVmRzZFUxQ2IwZERhWE5IUVZGUlFtYzNPSGRCVVRoRlJFRjNTMDFVUVROTlZGRXpUbFJWZVFwUFZFRjFRbWR2Y2tKblJVVkJXVTh2VFVGRlVVSkRRVTFJYldnd1pFaENlazlwT0haYU1td3dZVWhXYVV4dFRuWmlVemx2WkZka2JtRlhOVzVhYlVacUNscFVRVmxDWjI5eVFtZEZSVUZaVHk5TlFVVlNRa0Z2VFVORVNURk9la2wzVG5wUmVrMUhNRWREYVhOSFFWRlJRbWMzT0hkQlVrbEZXSGQ0WkdGSVVqQUtZMGhOTmt4NU9XNWhXRkp2WkZkSmRWa3lPWFJNTW1neFdqSmtjR0p0WkcxWlYwNXNUREowYkdOdE5XeGlTRTEwV1RJNWRHSllWblZoV0ZJMVRIazFiZ3BoV0ZKdlpGZEpkbVF5T1hsaE1scHpZak5rZWt3eVNqRmhWM2hyVEc1c2FHSlhlRUZqYlZadFkzazViMXBYUm10amVUbDBXVmRzZFUxRVowZERhWE5IQ2tGUlVVSm5OemgzUVZKTlJVdG5kMjlhYlZWNVdXMUpNazV0V214YVZGcHFUVzFGTTA0eVZtcFpNbGwzVDBkRmQwNVVRVFJhUjAweFRucFJlVTlIVVRBS1QwUm9iRnBFUVdoQ1oyOXlRbWRGUlVGWlR5OU5RVVZWUWtKTlRVVllaSFpqYlhSdFlrYzVNMWd5VW5Cak0wSm9aRWRPYjAxSFVVZERhWE5IUVZGUlFncG5OemgzUVZKVlJWWm5lRlZoU0ZJd1kwaE5Oa3g1T1c1aFdGSnZaRmRKZFZreU9YUk1NbWd4V2pKa2NHSnRaRzFaVjA1c1RESjBiR050Tld4aVNFMTBDbGt5T1hSaVdGWjFZVmhTTlV3eVJtcGtSMngyWW01TmRtTnVWblZqZVRoNlQxUkplazlVVlhoUFJFVjNUa001YUdSSVVteGlXRUl3WTNrNGVFMUNXVWNLUTJselIwRlJVVUpuTnpoM1FWSlpSVU5CZDBkalNGWnBZa2RzYWsxRldVZERhWE5IUVZGUlFtYzNPSGRCVW1kRlQwRjNNbU50Vm5kaWVuQnZaRmRrYmdwaFZ6VnVXbTFHYWxwVE9YSmFXRXAxV2xkNGVreFhUblppVnpFeFltMXNNR1ZVY0hsYVYxazJZMjFXYldONU9XOWFWMFpyWTNrNWRGbFhiSFZOU1VkTENrSm5iM0pDWjBWRlFXUmFOVUZuVVVOQ1NIZEZaV2RDTkVGSVdVRXpWREIzWVhOaVNFVlVTbXBIVWpSamJWZGpNMEZ4U2t0WWNtcGxVRXN6TDJnMGNIa0taME00Y0Rkdk5FRkJRVWRvVm10d1VVaDNRVUZDUVUxQlVucENSa0ZwUlVFMUsyVm9NR2Q0Tnl0VlVDdHRWRTFhTjJSWlIzUllRVWR2VFhJdmRUTkNMd294UVZSdlluVm5XbWx5WjBOSlJFMURkVzFNVDFCUWRtODVUbk13ZGtNNGVXSnlVWFEzWmxoSU5XZzVPR0ZqU21aR09XRlpWakZtZFUxQmIwZERRM0ZIQ2xOTk5EbENRVTFFUVRKclFVMUhXVU5OVVVNeFoyZG5SWFZaZEZZd2VHcFRRbEZHVEc1Q2VtSXlUbVJUVFZoWWJuSjNUbEZOWWs5emNXVnNaR3h3VkVjS2IxVnZkVWhOWjFRd1RtSnZUM1p3TnpST1kwTk5VVU41TnpsM1owVXlXV0o0V0VSYVZ6ZDVaVFYyT1dFNFZEY3liM3BrZVhsQlJ6TXJZa1pwZUVsSWFnb3djbmc0Tm1sdVZHUnJkWEZxY2poSk9UQnBkemx2YXowS0xTMHRMUzFGVGtRZ1EwVlNWRWxHU1VOQlZFVXRMUzB0TFFvPSJ9fX19"}], "timestampVerificationData":{"rfc3161Timestamps":[{"signedTimestamp":"MIICyjADAgEAMIICwQYJKoZIhvcNAQcCoIICsjCCAq4CAQMxDTALBglghkgBZQMEAgEwgbgGCyqGSIb3DQEJEAEEoIGoBIGlMIGiAgEBBgkrBgEEAYO/MAIwMTANBglghkgBZQMEAgEFAAQgve8SsvliP+kBGAv5uenHP7txgoqMJQ//kJyYCG+oIZQCFQCFqztCSeTNbThwd1EK8pfaDGvUzxgPMjAyNjEwMTkyMjMxMTNaMAMCAQGgMqQwMC4xFTATBgNVBAoTDHNpZ3N0b3JlLmRldjEVMBMGA1UEAxMMc2lnc3RvcmUtdHNhoAAxggHbMIIB1wIBATBRMDkxFTATBgNVBAoTDHNpZ3N0b3JlLmRldjEgMB4GA1UEAxMXc2lnc3RvcmUtdHNhLXNlbGZzaWduZWQCFDoTVC8MkGHuvMFDL8uKjosqI4sMMAsGCWCGSAFlAwQCAaCB/DAaBgkqhkiG9w0BCQMxDQYLKoZIhvcNAQkQAQQwHAYJKoZIhvcNAQkFMQ8XDTI2MTAxOTIyMzExM1owLwYJKoZIhvcNAQkEMSIEIABz+BxWlSmUQSkAqKq3rF9he+spJMqg2Knzgd2nAeimMIGOBgsqhkiG9w0BCRACLzF/MH0wezB5BCCF+Se8B6tiysO0Q1bBDvyBssaIP9p6uebYcNnROs0FtzBVMD2kOzA5MRUwEwYDVQQKEwxzaWdzdG9yZS5kZXYxIDAeBgNVBAMTF3NpZ3N0b3JlLXRzYS1zZWxmc2lnbmVkAhQ6E1QvDJBh7rzBQy/Lio6LKiOLDDAKBggqhkjOPQQDAgRnMGUCMDCtCa0AeNpbVDBgccxCXaS9qeyqeakv/2BZZVHiqKiCNrXU79GQ6q9FTWcg6JIxMgIxANVyI0gY8+W9UpPT0WZEBk0Fo2APA2MAPz5gH/ksPyYJIeHDeKqZjnLfkEacqtMAYw=="}]}}, "messageSignature":{"messageDigest":{"algorithm":"SHA2_256", "digest":"UKgD0ihI2KGo9tqt9b3lqtbZc1BiWSDKiV/AJDb11EU="}, "signature":"MEQCIBo+gqmoOjyG15QiRDQwwxA3n1Q1V/7uBk8TKbVydJd8AiBqMp/zJVdv+c7R3Hs6RQglNDBaafrX74TgnSC2tt7SLA=="}}
build/torch211-rocm71-x86_64-linux/mha.py ADDED
@@ -0,0 +1,332 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FlashAttention forward entry points backed by aiter's Composable-Kernel FMHA.
2
+
3
+ This is the compiled (HIP / Composable Kernel) counterpart to the Triton
4
+ ``aiter-flash-attn`` package. Only the *forward* pass is provided (dense and
5
+ variable-length), including support for learnable attention sinks via
6
+ ``window_size[2]`` / ``s_aux`` (e.g. gpt-oss). The op set ships exclusively
7
+ the CK ``mha_fwd`` / ``mha_varlen_fwd`` kernels, so the sink path is always
8
+ taken (no fall-through to a non-sink ASM kernel).
9
+ """
10
+
11
+ from typing import Optional, Tuple, Union
12
+
13
+ import torch
14
+ from torch import Tensor
15
+
16
+ from ._ops import ops
17
+
18
+
19
+ def _round_multiple(x: int, m: int) -> int:
20
+ return (x + m - 1) // m * m
21
+
22
+
23
+ def _maybe_pad_headdim(t: Tensor, target: int) -> Tensor:
24
+ og = t.size(-1)
25
+ if og == target:
26
+ return t
27
+ return torch.nn.functional.pad(t, [0, target - og])
28
+
29
+
30
+ def _unpack_window(window_size) -> Tuple[int, int, int]:
31
+ left = int(window_size[0])
32
+ right = int(window_size[1])
33
+ sink = int(window_size[2]) if len(window_size) == 3 else 0
34
+ return left, right, sink
35
+
36
+
37
+ def _prep_sink(s_aux: Optional[Tensor], nheads: int, device) -> Optional[Tensor]:
38
+ if s_aux is None:
39
+ return None
40
+ assert s_aux.device == device, "s_aux must be on the same device as q"
41
+ assert s_aux.shape[0] == nheads, "s_aux must have shape [nheads_q]"
42
+ if s_aux.dtype != torch.float32:
43
+ s_aux = s_aux.to(torch.float32)
44
+ return s_aux
45
+
46
+
47
+ def _effective_sink_size(sink_size: int, s_aux: Optional[Tensor]) -> int:
48
+ # The CK kernel only takes the sink path when sink_size > 0. Callers using
49
+ # the transformers attention interface pass the per-head sink logits as
50
+ # ``s_aux`` but leave ``window_size`` a 2-tuple (sink_size == 0), so enable
51
+ # a single learnable sink automatically when s_aux is given.
52
+ if s_aux is not None and sink_size == 0:
53
+ return 1
54
+ return sink_size
55
+
56
+
57
+ def flash_attn_func(
58
+ q: Tensor,
59
+ k: Tensor,
60
+ v: Tensor,
61
+ dropout_p: float = 0.0,
62
+ softmax_scale: Optional[float] = None,
63
+ causal: bool = False,
64
+ window_size: Tuple[int, ...] = (-1, -1, 0),
65
+ bias: Optional[Tensor] = None,
66
+ alibi_slopes: Optional[Tensor] = None,
67
+ return_lse: bool = False,
68
+ return_attn_probs: bool = False,
69
+ s_aux: Optional[Tensor] = None,
70
+ **kwargs,
71
+ ) -> Union[Tensor, Tuple[Tensor, ...]]:
72
+ """Dense FlashAttention forward.
73
+
74
+ ``q, k, v`` have shape ``(batch, seqlen, nheads, headdim)``. ``window_size``
75
+ is ``(left, right, sink_size)``; ``s_aux`` is an optional ``(nheads_q,)``
76
+ fp32 tensor of learnable sink logits. Extra keyword arguments (e.g. those
77
+ passed by the transformers flash-attention interface) are ignored.
78
+ """
79
+ if softmax_scale is None:
80
+ softmax_scale = q.shape[-1] ** (-0.5)
81
+ window_size_left, window_size_right, sink_size = _unpack_window(window_size)
82
+
83
+ nheads_q = q.size(2)
84
+ s_aux = _prep_sink(s_aux, nheads_q, q.device)
85
+ sink_size = _effective_sink_size(sink_size, s_aux)
86
+
87
+ head_q_og = q.size(-1)
88
+ head_v_og = v.size(-1)
89
+ head_q = _round_multiple(head_q_og, 8)
90
+ head_v = _round_multiple(head_v_og, 8)
91
+ q = _maybe_pad_headdim(q, head_q)
92
+ k = _maybe_pad_headdim(k, head_q)
93
+ v = _maybe_pad_headdim(v, head_v)
94
+
95
+ return_softmax = return_attn_probs and dropout_p > 0
96
+ out, softmax_lse, S_dmask, _ = ops.mha_fwd(
97
+ q,
98
+ k,
99
+ v,
100
+ dropout_p,
101
+ softmax_scale,
102
+ causal,
103
+ window_size_left,
104
+ window_size_right,
105
+ sink_size,
106
+ return_lse or return_attn_probs,
107
+ return_softmax,
108
+ None, # cu_seqlens_q
109
+ None, # cu_seqlens_kv
110
+ None, # out
111
+ bias,
112
+ alibi_slopes,
113
+ None, # q_descale
114
+ None, # k_descale
115
+ None, # v_descale
116
+ s_aux,
117
+ None, # gen
118
+ )
119
+
120
+ out = out[..., :head_v_og]
121
+ if not (return_lse or return_attn_probs):
122
+ return out
123
+ result = (out,)
124
+ if return_lse:
125
+ result = result + (softmax_lse,)
126
+ if return_attn_probs:
127
+ result = result + (S_dmask,)
128
+ return result
129
+
130
+
131
+ def flash_attn_varlen_func(
132
+ q: Tensor,
133
+ k: Tensor,
134
+ v: Tensor,
135
+ cu_seqlens_q: Tensor,
136
+ cu_seqlens_k: Tensor,
137
+ max_seqlen_q: int,
138
+ max_seqlen_k: int,
139
+ dropout_p: float = 0.0,
140
+ softmax_scale: Optional[float] = None,
141
+ causal: bool = False,
142
+ window_size: Tuple[int, ...] = (-1, -1, 0),
143
+ bias: Optional[Tensor] = None,
144
+ alibi_slopes: Optional[Tensor] = None,
145
+ return_lse: bool = False,
146
+ return_attn_probs: bool = False,
147
+ block_table: Optional[Tensor] = None,
148
+ min_seqlen_q: int = 0,
149
+ logits_soft_cap: float = 0.0,
150
+ zero_tensors: bool = False,
151
+ cu_seqlens_q_padded: Optional[Tensor] = None,
152
+ cu_seqlens_k_padded: Optional[Tensor] = None,
153
+ s_aux: Optional[Tensor] = None,
154
+ **kwargs,
155
+ ) -> Union[Tensor, Tuple[Tensor, ...]]:
156
+ """Variable-length / packed FlashAttention forward.
157
+
158
+ ``q`` has shape ``(total_q, nheads, headdim)``; ``cu_seqlens_*`` are int32
159
+ cumulative offsets of shape ``(batch + 1,)``. Sinks are supported through
160
+ ``window_size[2]`` / ``s_aux`` as in :func:`flash_attn_func`. Extra keyword
161
+ arguments (e.g. from the transformers flash-attention interface) are ignored.
162
+ """
163
+ if softmax_scale is None:
164
+ softmax_scale = q.shape[-1] ** (-0.5)
165
+ window_size_left, window_size_right, sink_size = _unpack_window(window_size)
166
+
167
+ nheads_q = q.size(1)
168
+ s_aux = _prep_sink(s_aux, nheads_q, q.device)
169
+ sink_size = _effective_sink_size(sink_size, s_aux)
170
+
171
+ head_q_og = q.size(-1)
172
+ head_v_og = v.size(-1)
173
+ head_q = _round_multiple(head_q_og, 8)
174
+ head_v = _round_multiple(head_v_og, 8)
175
+ q = _maybe_pad_headdim(q, head_q)
176
+ k = _maybe_pad_headdim(k, head_q)
177
+ v = _maybe_pad_headdim(v, head_v)
178
+
179
+ return_softmax = return_attn_probs and dropout_p > 0
180
+ out, softmax_lse, S_dmask, _ = ops.mha_varlen_fwd(
181
+ q,
182
+ k,
183
+ v,
184
+ cu_seqlens_q,
185
+ cu_seqlens_k,
186
+ max_seqlen_q,
187
+ max_seqlen_k,
188
+ min_seqlen_q,
189
+ dropout_p,
190
+ softmax_scale,
191
+ logits_soft_cap,
192
+ zero_tensors,
193
+ causal,
194
+ window_size_left,
195
+ window_size_right,
196
+ sink_size,
197
+ return_lse or return_attn_probs,
198
+ return_softmax,
199
+ None, # out
200
+ block_table,
201
+ bias,
202
+ alibi_slopes,
203
+ None, # q_descale
204
+ None, # k_descale
205
+ None, # v_descale
206
+ None, # gen
207
+ cu_seqlens_q_padded,
208
+ cu_seqlens_k_padded,
209
+ s_aux,
210
+ )
211
+
212
+ out = out[..., :head_v_og]
213
+ if not (return_lse or return_attn_probs):
214
+ return out
215
+ result = (out,)
216
+ if return_lse:
217
+ result = result + (softmax_lse,)
218
+ if return_attn_probs:
219
+ result = result + (S_dmask,)
220
+ return result
221
+
222
+
223
+ def _append_to_contiguous_cache(k_cache, v_cache, k, v, cache_seqlens, Sq):
224
+ """In-place append of new k/v into a contiguous cache at per-batch offsets."""
225
+ B = k_cache.shape[0]
226
+ if isinstance(cache_seqlens, int):
227
+ cache_seqlens = torch.full((B,), cache_seqlens, device=k_cache.device, dtype=torch.long)
228
+ pos = cache_seqlens.view(B, 1).to(torch.long) + torch.arange(Sq, device=k_cache.device).view(1, Sq)
229
+ bidx = torch.arange(B, device=k_cache.device).view(B, 1).expand(B, Sq)
230
+ k_cache[bidx, pos] = k.to(k_cache.dtype)
231
+ v_cache[bidx, pos] = v.to(v_cache.dtype)
232
+
233
+
234
+ def flash_attn_with_kvcache(
235
+ q: Tensor,
236
+ k_cache: Tensor,
237
+ v_cache: Tensor,
238
+ k: Optional[Tensor] = None,
239
+ v: Optional[Tensor] = None,
240
+ rotary_cos: Optional[Tensor] = None,
241
+ rotary_sin: Optional[Tensor] = None,
242
+ cache_seqlens=None,
243
+ block_table: Optional[Tensor] = None,
244
+ softmax_scale: Optional[float] = None,
245
+ causal: bool = True,
246
+ window_size: Tuple[int, ...] = (-1, -1, 0),
247
+ s_aux: Optional[Tensor] = None,
248
+ return_lse: bool = False,
249
+ **kwargs,
250
+ ) -> Union[Tensor, Tuple[Tensor, ...]]:
251
+ """FlashAttention against a KV cache, built on the CK split-KV kernel.
252
+
253
+ ``q`` is ``(batch, seqlen_q, nheads, headdim)``. For a contiguous cache,
254
+ ``k_cache``/``v_cache`` are ``(batch, seqlen_cache, nheads_k, headdim)`` and
255
+ ``cache_seqlens`` (int or ``(batch,)``) is the number of valid cache entries
256
+ *before* appending. If ``k``/``v`` (the new tokens) are given, they are
257
+ written into the cache in place. The query attends to the valid prefix
258
+ (``cache_seqlens`` + ``seqlen_q``) with bottom-right causal alignment.
259
+
260
+ Notes / current limitations:
261
+ - Rotary embedding is not applied here; pass ``q``/``k`` already rotated
262
+ (``rotary_cos``/``rotary_sin`` are rejected).
263
+ - Paged caches are addressed via ``block_table`` (forwarded to the split-KV
264
+ varlen kernel); in-cache append for paged layouts must be done by the
265
+ caller (pass ``k=v=None``).
266
+ """
267
+ if rotary_cos is not None or rotary_sin is not None:
268
+ raise NotImplementedError(
269
+ "flash_attn_with_kvcache does not apply rotary; rotate q/k beforehand."
270
+ )
271
+ if softmax_scale is None:
272
+ softmax_scale = q.shape[-1] ** (-0.5)
273
+ window_size_left, window_size_right, sink_size = _unpack_window(window_size)
274
+ B, Sq, Hq, D = q.shape
275
+ s_aux = _prep_sink(s_aux, Hq, q.device)
276
+ sink_size = _effective_sink_size(sink_size, s_aux)
277
+
278
+ paged = block_table is not None
279
+
280
+ # ---- append new tokens into the cache (contiguous layout only) ----
281
+ if k is not None and not paged:
282
+ assert cache_seqlens is not None, "cache_seqlens required to append into the cache"
283
+ _append_to_contiguous_cache(k_cache, v_cache, k, v, cache_seqlens, Sq)
284
+ elif k is not None and paged:
285
+ raise NotImplementedError(
286
+ "in-cache append for paged layouts is not supported; append externally "
287
+ "and call with k=v=None."
288
+ )
289
+
290
+ appended = 0 if k is None else Sq
291
+
292
+ # ---- key lengths ----
293
+ if cache_seqlens is None:
294
+ klens = torch.full((B,), k_cache.shape[1], device=q.device, dtype=torch.long)
295
+ elif isinstance(cache_seqlens, int):
296
+ klens = torch.full((B,), cache_seqlens + appended, device=q.device, dtype=torch.long)
297
+ else:
298
+ klens = cache_seqlens.to(q.device, torch.long) + appended
299
+
300
+ uniform = bool((klens == klens[0]).all().item())
301
+
302
+ # ---- fast path: contiguous cache, uniform length -> dense kernel ----
303
+ if not paged and uniform:
304
+ L = int(klens[0].item())
305
+ return flash_attn_func(
306
+ q, k_cache[:, :L], v_cache[:, :L],
307
+ softmax_scale=softmax_scale, causal=causal,
308
+ window_size=(window_size_left, window_size_right, sink_size),
309
+ s_aux=s_aux, return_lse=return_lse,
310
+ )
311
+
312
+ # ---- general path: variable cache lengths -> split-KV varlen kernel ----
313
+ cu_q = torch.arange(0, (B + 1) * Sq, Sq, device=q.device, dtype=torch.int32)
314
+ cu_k = torch.zeros(B + 1, device=q.device, dtype=torch.int32)
315
+ cu_k[1:] = torch.cumsum(klens, 0).to(torch.int32)
316
+ q_packed = q.reshape(B * Sq, Hq, D)
317
+ if paged:
318
+ k_packed, v_packed = k_cache, v_cache # kernel gathers via block_table
319
+ else:
320
+ k_packed = torch.cat([k_cache[b, : int(klens[b])] for b in range(B)], dim=0)
321
+ v_packed = torch.cat([v_cache[b, : int(klens[b])] for b in range(B)], dim=0)
322
+
323
+ out = flash_attn_varlen_func(
324
+ q_packed, k_packed, v_packed, cu_q, cu_k, Sq, int(klens.max().item()),
325
+ softmax_scale=softmax_scale, causal=causal,
326
+ window_size=(window_size_left, window_size_right, sink_size),
327
+ block_table=block_table, s_aux=s_aux, return_lse=return_lse,
328
+ )
329
+ if return_lse:
330
+ out, lse = out
331
+ return out.reshape(B, Sq, Hq, D), lse
332
+ return out.reshape(B, Sq, Hq, D)
build/torch211-rocm71-x86_64-linux/symbols.json ADDED
@@ -0,0 +1,349 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "schema_version": 1,
3
+ "module": "_aiter_flash_attn_ck_rocm_fe2bb66",
4
+ "functions": [
5
+ {
6
+ "name": "flash_attn_func",
7
+ "qualname": "flash_attn_func",
8
+ "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha",
9
+ "kind": "function",
10
+ "doc": "Dense FlashAttention forward.\n\n``q, k, v`` have shape ``(batch, seqlen, nheads, headdim)``. ``window_size``\nis ``(left, right, sink_size)``; ``s_aux`` is an optional ``(nheads_q,)``\nfp32 tensor of learnable sink logits. Extra keyword arguments (e.g. those\npassed by the transformers flash-attention interface) are ignored.",
11
+ "signature": {
12
+ "parameters": [
13
+ {
14
+ "name": "q",
15
+ "kind": "POSITIONAL_OR_KEYWORD",
16
+ "default": null,
17
+ "annotation": "torch.Tensor"
18
+ },
19
+ {
20
+ "name": "k",
21
+ "kind": "POSITIONAL_OR_KEYWORD",
22
+ "default": null,
23
+ "annotation": "torch.Tensor"
24
+ },
25
+ {
26
+ "name": "v",
27
+ "kind": "POSITIONAL_OR_KEYWORD",
28
+ "default": null,
29
+ "annotation": "torch.Tensor"
30
+ },
31
+ {
32
+ "name": "dropout_p",
33
+ "kind": "POSITIONAL_OR_KEYWORD",
34
+ "default": "0.0",
35
+ "annotation": "float"
36
+ },
37
+ {
38
+ "name": "softmax_scale",
39
+ "kind": "POSITIONAL_OR_KEYWORD",
40
+ "default": "None",
41
+ "annotation": "float | None"
42
+ },
43
+ {
44
+ "name": "causal",
45
+ "kind": "POSITIONAL_OR_KEYWORD",
46
+ "default": "False",
47
+ "annotation": "bool"
48
+ },
49
+ {
50
+ "name": "window_size",
51
+ "kind": "POSITIONAL_OR_KEYWORD",
52
+ "default": "(-1, -1, 0)",
53
+ "annotation": "Tuple[int, ...]"
54
+ },
55
+ {
56
+ "name": "bias",
57
+ "kind": "POSITIONAL_OR_KEYWORD",
58
+ "default": "None",
59
+ "annotation": "torch.Tensor | None"
60
+ },
61
+ {
62
+ "name": "alibi_slopes",
63
+ "kind": "POSITIONAL_OR_KEYWORD",
64
+ "default": "None",
65
+ "annotation": "torch.Tensor | None"
66
+ },
67
+ {
68
+ "name": "return_lse",
69
+ "kind": "POSITIONAL_OR_KEYWORD",
70
+ "default": "False",
71
+ "annotation": "bool"
72
+ },
73
+ {
74
+ "name": "return_attn_probs",
75
+ "kind": "POSITIONAL_OR_KEYWORD",
76
+ "default": "False",
77
+ "annotation": "bool"
78
+ },
79
+ {
80
+ "name": "s_aux",
81
+ "kind": "POSITIONAL_OR_KEYWORD",
82
+ "default": "None",
83
+ "annotation": "torch.Tensor | None"
84
+ },
85
+ {
86
+ "name": "kwargs",
87
+ "kind": "VAR_KEYWORD",
88
+ "default": null,
89
+ "annotation": null
90
+ }
91
+ ],
92
+ "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]"
93
+ }
94
+ },
95
+ {
96
+ "name": "flash_attn_varlen_func",
97
+ "qualname": "flash_attn_varlen_func",
98
+ "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha",
99
+ "kind": "function",
100
+ "doc": "Variable-length / packed FlashAttention forward.\n\n``q`` has shape ``(total_q, nheads, headdim)``; ``cu_seqlens_*`` are int32\ncumulative offsets of shape ``(batch + 1,)``. Sinks are supported through\n``window_size[2]`` / ``s_aux`` as in :func:`flash_attn_func`. Extra keyword\narguments (e.g. from the transformers flash-attention interface) are ignored.",
101
+ "signature": {
102
+ "parameters": [
103
+ {
104
+ "name": "q",
105
+ "kind": "POSITIONAL_OR_KEYWORD",
106
+ "default": null,
107
+ "annotation": "torch.Tensor"
108
+ },
109
+ {
110
+ "name": "k",
111
+ "kind": "POSITIONAL_OR_KEYWORD",
112
+ "default": null,
113
+ "annotation": "torch.Tensor"
114
+ },
115
+ {
116
+ "name": "v",
117
+ "kind": "POSITIONAL_OR_KEYWORD",
118
+ "default": null,
119
+ "annotation": "torch.Tensor"
120
+ },
121
+ {
122
+ "name": "cu_seqlens_q",
123
+ "kind": "POSITIONAL_OR_KEYWORD",
124
+ "default": null,
125
+ "annotation": "torch.Tensor"
126
+ },
127
+ {
128
+ "name": "cu_seqlens_k",
129
+ "kind": "POSITIONAL_OR_KEYWORD",
130
+ "default": null,
131
+ "annotation": "torch.Tensor"
132
+ },
133
+ {
134
+ "name": "max_seqlen_q",
135
+ "kind": "POSITIONAL_OR_KEYWORD",
136
+ "default": null,
137
+ "annotation": "int"
138
+ },
139
+ {
140
+ "name": "max_seqlen_k",
141
+ "kind": "POSITIONAL_OR_KEYWORD",
142
+ "default": null,
143
+ "annotation": "int"
144
+ },
145
+ {
146
+ "name": "dropout_p",
147
+ "kind": "POSITIONAL_OR_KEYWORD",
148
+ "default": "0.0",
149
+ "annotation": "float"
150
+ },
151
+ {
152
+ "name": "softmax_scale",
153
+ "kind": "POSITIONAL_OR_KEYWORD",
154
+ "default": "None",
155
+ "annotation": "float | None"
156
+ },
157
+ {
158
+ "name": "causal",
159
+ "kind": "POSITIONAL_OR_KEYWORD",
160
+ "default": "False",
161
+ "annotation": "bool"
162
+ },
163
+ {
164
+ "name": "window_size",
165
+ "kind": "POSITIONAL_OR_KEYWORD",
166
+ "default": "(-1, -1, 0)",
167
+ "annotation": "Tuple[int, ...]"
168
+ },
169
+ {
170
+ "name": "bias",
171
+ "kind": "POSITIONAL_OR_KEYWORD",
172
+ "default": "None",
173
+ "annotation": "torch.Tensor | None"
174
+ },
175
+ {
176
+ "name": "alibi_slopes",
177
+ "kind": "POSITIONAL_OR_KEYWORD",
178
+ "default": "None",
179
+ "annotation": "torch.Tensor | None"
180
+ },
181
+ {
182
+ "name": "return_lse",
183
+ "kind": "POSITIONAL_OR_KEYWORD",
184
+ "default": "False",
185
+ "annotation": "bool"
186
+ },
187
+ {
188
+ "name": "return_attn_probs",
189
+ "kind": "POSITIONAL_OR_KEYWORD",
190
+ "default": "False",
191
+ "annotation": "bool"
192
+ },
193
+ {
194
+ "name": "block_table",
195
+ "kind": "POSITIONAL_OR_KEYWORD",
196
+ "default": "None",
197
+ "annotation": "torch.Tensor | None"
198
+ },
199
+ {
200
+ "name": "min_seqlen_q",
201
+ "kind": "POSITIONAL_OR_KEYWORD",
202
+ "default": "0",
203
+ "annotation": "int"
204
+ },
205
+ {
206
+ "name": "logits_soft_cap",
207
+ "kind": "POSITIONAL_OR_KEYWORD",
208
+ "default": "0.0",
209
+ "annotation": "float"
210
+ },
211
+ {
212
+ "name": "zero_tensors",
213
+ "kind": "POSITIONAL_OR_KEYWORD",
214
+ "default": "False",
215
+ "annotation": "bool"
216
+ },
217
+ {
218
+ "name": "cu_seqlens_q_padded",
219
+ "kind": "POSITIONAL_OR_KEYWORD",
220
+ "default": "None",
221
+ "annotation": "torch.Tensor | None"
222
+ },
223
+ {
224
+ "name": "cu_seqlens_k_padded",
225
+ "kind": "POSITIONAL_OR_KEYWORD",
226
+ "default": "None",
227
+ "annotation": "torch.Tensor | None"
228
+ },
229
+ {
230
+ "name": "s_aux",
231
+ "kind": "POSITIONAL_OR_KEYWORD",
232
+ "default": "None",
233
+ "annotation": "torch.Tensor | None"
234
+ },
235
+ {
236
+ "name": "kwargs",
237
+ "kind": "VAR_KEYWORD",
238
+ "default": null,
239
+ "annotation": null
240
+ }
241
+ ],
242
+ "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]"
243
+ }
244
+ },
245
+ {
246
+ "name": "flash_attn_with_kvcache",
247
+ "qualname": "flash_attn_with_kvcache",
248
+ "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha",
249
+ "kind": "function",
250
+ "doc": "FlashAttention against a KV cache, built on the CK split-KV kernel.\n\n``q`` is ``(batch, seqlen_q, nheads, headdim)``. For a contiguous cache,\n``k_cache``/``v_cache`` are ``(batch, seqlen_cache, nheads_k, headdim)`` and\n``cache_seqlens`` (int or ``(batch,)``) is the number of valid cache entries\n*before* appending. If ``k``/``v`` (the new tokens) are given, they are\nwritten into the cache in place. The query attends to the valid prefix\n(``cache_seqlens`` + ``seqlen_q``) with bottom-right causal alignment.\n\nNotes / current limitations:\n- Rotary embedding is not applied here; pass ``q``/``k`` already rotated\n (``rotary_cos``/``rotary_sin`` are rejected).\n- Paged caches are addressed via ``block_table`` (forwarded to the split-KV\n varlen kernel); in-cache append for paged layouts must be done by the\n caller (pass ``k=v=None``).",
251
+ "signature": {
252
+ "parameters": [
253
+ {
254
+ "name": "q",
255
+ "kind": "POSITIONAL_OR_KEYWORD",
256
+ "default": null,
257
+ "annotation": "torch.Tensor"
258
+ },
259
+ {
260
+ "name": "k_cache",
261
+ "kind": "POSITIONAL_OR_KEYWORD",
262
+ "default": null,
263
+ "annotation": "torch.Tensor"
264
+ },
265
+ {
266
+ "name": "v_cache",
267
+ "kind": "POSITIONAL_OR_KEYWORD",
268
+ "default": null,
269
+ "annotation": "torch.Tensor"
270
+ },
271
+ {
272
+ "name": "k",
273
+ "kind": "POSITIONAL_OR_KEYWORD",
274
+ "default": "None",
275
+ "annotation": "torch.Tensor | None"
276
+ },
277
+ {
278
+ "name": "v",
279
+ "kind": "POSITIONAL_OR_KEYWORD",
280
+ "default": "None",
281
+ "annotation": "torch.Tensor | None"
282
+ },
283
+ {
284
+ "name": "rotary_cos",
285
+ "kind": "POSITIONAL_OR_KEYWORD",
286
+ "default": "None",
287
+ "annotation": "torch.Tensor | None"
288
+ },
289
+ {
290
+ "name": "rotary_sin",
291
+ "kind": "POSITIONAL_OR_KEYWORD",
292
+ "default": "None",
293
+ "annotation": "torch.Tensor | None"
294
+ },
295
+ {
296
+ "name": "cache_seqlens",
297
+ "kind": "POSITIONAL_OR_KEYWORD",
298
+ "default": "None",
299
+ "annotation": null
300
+ },
301
+ {
302
+ "name": "block_table",
303
+ "kind": "POSITIONAL_OR_KEYWORD",
304
+ "default": "None",
305
+ "annotation": "torch.Tensor | None"
306
+ },
307
+ {
308
+ "name": "softmax_scale",
309
+ "kind": "POSITIONAL_OR_KEYWORD",
310
+ "default": "None",
311
+ "annotation": "float | None"
312
+ },
313
+ {
314
+ "name": "causal",
315
+ "kind": "POSITIONAL_OR_KEYWORD",
316
+ "default": "True",
317
+ "annotation": "bool"
318
+ },
319
+ {
320
+ "name": "window_size",
321
+ "kind": "POSITIONAL_OR_KEYWORD",
322
+ "default": "(-1, -1, 0)",
323
+ "annotation": "Tuple[int, ...]"
324
+ },
325
+ {
326
+ "name": "s_aux",
327
+ "kind": "POSITIONAL_OR_KEYWORD",
328
+ "default": "None",
329
+ "annotation": "torch.Tensor | None"
330
+ },
331
+ {
332
+ "name": "return_lse",
333
+ "kind": "POSITIONAL_OR_KEYWORD",
334
+ "default": "False",
335
+ "annotation": "bool"
336
+ },
337
+ {
338
+ "name": "kwargs",
339
+ "kind": "VAR_KEYWORD",
340
+ "default": null,
341
+ "annotation": null
342
+ }
343
+ ],
344
+ "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]"
345
+ }
346
+ }
347
+ ],
348
+ "layers": []
349
+ }
build/torch211-rocm72-x86_64-linux/__init__.py ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """AITER Composable-Kernel FlashAttention kernels for AMD ROCm (compiled HIP).
2
+
3
+ Compiled (Composable Kernel / HIP) counterpart to the Triton-based
4
+ ``aiter-flash-attn`` package, repackaged from the CK FMHA forward path of the
5
+ `ROCm/aiter <https://github.com/ROCm/aiter>`_ project as a self-contained
6
+ Hugging Face Hub kernel. Exposes the FlashAttention forward entry points with
7
+ learnable attention sink support (e.g. gpt-oss).
8
+ """
9
+
10
+ from .mha import (
11
+ flash_attn_func,
12
+ flash_attn_varlen_func,
13
+ flash_attn_with_kvcache,
14
+ )
15
+
16
+
17
+ __kernel_metadata__ = {
18
+ "license": "mit",
19
+ }
20
+
21
+
22
+ __all__ = [
23
+ "__kernel_metadata__",
24
+ "flash_attn_func",
25
+ "flash_attn_varlen_func",
26
+ "flash_attn_with_kvcache",
27
+ ]
build/torch211-rocm72-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3c2f41c2913536638920d0d10d5ab4b4337639df8803362367d4edde7020b610
3
+ size 192185696
build/torch211-rocm72-x86_64-linux/_ops.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _aiter_flash_attn_ck_rocm_fe2bb66
3
+ ops = torch.ops._aiter_flash_attn_ck_rocm_fe2bb66
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ """
7
+ Prefix op by namespace.
8
+ """
9
+ return f"_aiter_flash_attn_ck_rocm_fe2bb66::{op_name}"
build/torch211-rocm72-x86_64-linux/metadata.json ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "aiter-flash-attn-ck",
3
+ "id": "_aiter_flash_attn_ck_rocm_fe2bb66",
4
+ "version": 1,
5
+ "kernels-minver": "0.14.0",
6
+ "license": "MIT",
7
+ "python-depends": [],
8
+ "kernel-depends": [],
9
+ "backend": {
10
+ "type": "rocm",
11
+ "archs": [
12
+ "gfx942",
13
+ "gfx950"
14
+ ]
15
+ },
16
+ "digest": {
17
+ "algorithm": "sha256",
18
+ "files": {
19
+ "__init__.py": "7wj1GTm2bqTLLCyT4xLuAk/+JWYrhINjY/ODMNWhgZ0=",
20
+ "_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so": "PC9BwpE1NmOJINDRDVq0tDN2Od+IAzYjZ9Tt3nAgthA=",
21
+ "_ops.py": "4WkdYDWWJp60zep+oW/5ym9vIeVMNfrgRCkP3o/w7yU=",
22
+ "mha.py": "zCAjY4O+DiCzmEL+W9sqXxZaqjsj4RJbWFcyPpUZejk=",
23
+ "symbols.json": "rhnF4H8Jsb6I+YPxIVFMgnLcTQJODEXbZ7ge77NZXAU="
24
+ }
25
+ },
26
+ "provenance": {
27
+ "kernel-builder": {
28
+ "version": "0.18.0-dev0",
29
+ "commit": "d054906a13710b811b96697b4635f03020842733",
30
+ "dirty": false
31
+ },
32
+ "kernel": {
33
+ "commit": "fe2bb66fee6c2a77eccf08a0508dc57428d488ed",
34
+ "dirty": false
35
+ }
36
+ }
37
+ }
build/torch211-rocm72-x86_64-linux/metadata.json.sigstore ADDED
@@ -0,0 +1 @@
 
 
1
+ {"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json", "verificationMaterial":{"certificate":{"rawBytes":"MIIHKzCCBrGgAwIBAgIUQTtf3Iyvqs7McLpN4yWfDq9ATiUwCgYIKoZIzj0EAwMwNzEVMBMGA1UEChMMc2lnc3RvcmUuZGV2MR4wHAYDVQQDExVzaWdzdG9yZS1pbnRlcm1lZGlhdGUwHhcNMjYxMDE5MjIzMTE0WhcNMjYxMDE5MjI0MTE0WjAAMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE3t4nw4ZHrBQ3CbTfkKAwLntof/04udWnGPHoBy22SWv0MFnWlFpH3ZxJg5xRTPsjhFBq+fbV9SsvCvDT14AWx6OCBdAwggXMMA4GA1UdDwEB/wQEAwIHgDATBgNVHSUEDDAKBggrBgEFBQcDAzAfBgNVHSMEGDAWgBTf0+nPViQRlvmo2OkoVaLGLhhkPzBrBgNVHREBAf8EYTBfhl1odHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkvLmdpdGh1Yi93b3JrZmxvd3MvYnVpbGQueWFtbEByZWZzL2hlYWRzL21haW4wOQYKKwYBBAGDvzABAQQraHR0cHM6Ly90b2tlbi5hY3Rpb25zLmdpdGh1YnVzZXJjb250ZW50LmNvbTAfBgorBgEEAYO/MAECBBF3b3JrZmxvd19kaXNwYXRjaDA2BgorBgEEAYO/MAEDBChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMBMGCisGAQQBg78wAQQEBUJ1aWxkMCsGCisGAQQBg78wAQUEHWh1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5MB0GCisGAQQBg78wAQYED3JlZnMvaGVhZHMvbWFpbjA7BgorBgEEAYO/MAEIBC0MK2h0dHBzOi8vdG9rZW4uYWN0aW9ucy5naXRodWJ1c2VyY29udGVudC5jb20wbQYKKwYBBAGDvzABCQRfDF1odHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkvLmdpdGh1Yi93b3JrZmxvd3MvYnVpbGQueWFtbEByZWZzL2hlYWRzL21haW4wOAYKKwYBBAGDvzABCgQqDChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMBsGCisGAQQBg78wAQsEDQwLc2VsZi1ob3N0ZWQwQAYKKwYBBAGDvzABDAQyDDBodHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkwOAYKKwYBBAGDvzABDQQqDChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMB8GCisGAQQBg78wAQ4EEQwPcmVmcy9oZWFkcy9tYWluMBoGCisGAQQBg78wAQ8EDAwKMTA3MTQ3NTUyOTAuBgorBgEEAYO/MAEQBCAMHmh0dHBzOi8vZ2l0aHViLmNvbS9odWdnaW5nZmFjZTAYBgorBgEEAYO/MAERBAoMCDI1NzIwNzQzMG0GCisGAQQBg78wARIEXwxdaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5Ly5naXRodWIvd29ya2Zsb3dzL2J1aWxkLnlhbWxAcmVmcy9oZWFkcy9tYWluMDgGCisGAQQBg78wARMEKgwoZmUyYmI2NmZlZTZjMmE3N2VjY2YwOGEwNTA4ZGM1NzQyOGQ0ODhlZDAhBgorBgEEAYO/MAEUBBMMEXdvcmtmbG93X2Rpc3BhdGNoMGQGCisGAQQBg78wARUEVgxUaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5L2FjdGlvbnMvcnVucy8zOTIzOTUxODEwNC9hdHRlbXB0cy8xMBYGCisGAQQBg78wARYECAwGcHVibGljMEYGCisGAQQBg78wARgEOAw2cmVwbzpodWdnaW5nZmFjZS9rZXJuZWxzLWNvbW11bml0eTpyZWY6cmVmcy9oZWFkcy9tYWluMIGJBgorBgEEAdZ5AgQCBHsEeQB3AHUA3T0wasbHETJjGR4cmWc3AqJKXrjePK3/h4pygC8p7o4AAAGhVkpUpwAABAMARjBEAiA1C1YmkJfAITRpzy5HtV9VCMYhGsoyd5hZgpWPpvfVKQIgOVIL0hfBTYj6gt4xl1xqgUEMiH5fkUfg3ZqFxFEBSRAwCgYIKoZIzj0EAwMDaAAwZQIxAJa4QXsPL81L+vXhGGcrZ1zELYnOUe8fmWuz/GLBPTq/v9rdnXKIzxp1A6bIWU8zowIwHGaQvnlFluvNCg6iFIDlDH///B5I/d+ZJwxA3O9O0q7tZggFRCSR0TX4b4K0pfeQ"}, "tlogEntries":[{"logIndex":"3327211583", "logId":{"keyId":"wNI9atQGlz+VWfO6LRygH4QUfY/8W4RFwiT5i5WRgB0="}, "kindVersion":{"kind":"hashedrekord", "version":"0.0.1"}, "integratedTime":"1792449074", "inclusionPromise":{"signedEntryTimestamp":"MEUCIAr7jCNtRJxMlzr1cB+bYl7IKogmxEEEVe2YNbLgJHA8AiEAiDYLQfiwGECJu77/mDsLBPN+ENXHui3Cb6O8P9ck/H0="}, "inclusionProof":{"logIndex":"3205307321", "rootHash":"+fepLxpPOkncVXhGVS4DEMtC08T30oRyfG2VFcMxDGk=", "treeSize":"3205307355", "hashes":["c7VQLPQB2GvvWkc+o6TggbQrbw9Fpr+eCKJewVrBtS8=", "N+iK1dAfNviVQ3nNHztMlyqFa74NPSlTSceuPLmAmTw=", "2wpyyp1ow0otcAGP3cCZwWLmF9FNsuEklXqugfMzXy8=", "JCSa700+gdDu+84caCYaG3zOO7mrMW3EPEh9Uwh91gU=", "mmy8PbTWegE54CQglLSVqGow9D3xL3rqWVCm4og3U2E=", "ZFFNXt17Kn8H1nfr6Yedo+Gde+FkiTYwyKxIfTBJEhs=", "kb9vIWaHg98rThNcZN0YfYNvf4CBr8OBzgh2Qa/FH2A=", "ykqNcyTAH4N/u/MuimGrj8+/P23oqcyJgXQ8Hd4uapU=", "AJlvaM7WQKZg0FAjSdh7RzJXyGKaOW5DVz46FbaUPZE=", "t4Yy4kaoiJ3+VosqXw5Ogmg+A7p/E6y8+JhqWbalMbI=", "5egUlZ4Pi0Ix/qLyCDXocsWywJCHqrde/S06vOZvlOM=", "QsXufUpoKtSjKAB19K1G00umQpWvPHlBNuRGFiiVjQQ=", "QIMkwQbwAewhLJd//5FFwI3SnyNQBGjCgJrelfG8vhc=", "Et17CACUv9QN64irUJuBjsrgRGkGnnJ9xacRV/aBdwQ=", "lqgEwkxhDMs/gdIAE4lmjfMwg7nDmv7lda6O0LGliCw=", "HovTPBWzoErvUg1u5sIi3x1ZonZ7yJekVsBrmpomCIA=", "G+nk7mgE5h6dUmv2UY7NDJrY3WvrZY31raaO1AETt+o=", "qlTWNDzERnTBQVcHHXXVPjiUP0SsikGxfcxVlgqVFDo=", "gg9GNApNke3+zNhl4tsWtmIycCFrOMCB3lys6XuSQNQ=", "NlOclldmKPA1tMKIskqHAwuQDYHp31YwpK+OOMrMccc=", "qxzHanAzz57SDdmJe0B7bJK72NTIBbwEMGBBKvDOROw=", "xH/DCseLHr9eKoYT8qsORZK7zVdEGYWHuVtsVrD95wY="], "checkpoint":{"envelope":"rekor.sigstore.dev - 1193050959916656506\n3205307355\n+fepLxpPOkncVXhGVS4DEMtC08T30oRyfG2VFcMxDGk=\n\n— rekor.sigstore.dev wNI9ajBEAiBdd0/8rQRLHE4cgX8ENMZ0/RCQPKa4qsZYGkjcthEV3AIgf8uLH7bF4jGMwAoMMQPLM1CGWjMYNWqJw3068ntJIiY=\n"}}, "canonicalizedBody":"eyJhcGlWZXJzaW9uIjoiMC4wLjEiLCJraW5kIjoiaGFzaGVkcmVrb3JkIiwic3BlYyI6eyJkYXRhIjp7Imhhc2giOnsiYWxnb3JpdGhtIjoic2hhMjU2IiwidmFsdWUiOiJjOGJkZjdhN2I0MWRlNTdiZDMyY2Q4NzdhNWRmODNkOTJkNTk2Y2MwMWYzYTM2ZTE1NTEyMjIxZjA4NWU2NGYxIn19LCJzaWduYXR1cmUiOnsiY29udGVudCI6Ik1FVUNJUUN6cEsrZGZSeXdLV3RGVVZHVDB5RXFkcDBOdnNBRXU2UEtLWWVNbm0wOXFRSWdJVmNVcUVnUmVJcDdtd0NMOEFsTkFocThZL3hvL012c09mV2RiQ3Z1QVlZPSIsInB1YmxpY0tleSI6eyJjb250ZW50IjoiTFMwdExTMUNSVWRKVGlCRFJWSlVTVVpKUTBGVVJTMHRMUzB0Q2sxSlNVaExla05EUW5KSFowRjNTVUpCWjBsVlVWUjBaak5KZVhaeGN6ZE5ZMHh3VGpSNVYyWkVjVGxCVkdsVmQwTm5XVWxMYjFwSmVtb3dSVUYzVFhjS1RucEZWazFDVFVkQk1WVkZRMmhOVFdNeWJHNWpNMUoyWTIxVmRWcEhWakpOVWpSM1NFRlpSRlpSVVVSRmVGWjZZVmRrZW1SSE9YbGFVekZ3WW01U2JBcGpiVEZzV2tkc2FHUkhWWGRJYUdOT1RXcFplRTFFUlRWTmFrbDZUVlJGTUZkb1kwNU5hbGw0VFVSRk5VMXFTVEJOVkVVd1YycEJRVTFHYTNkRmQxbElDa3R2V2tsNmFqQkRRVkZaU1V0dldrbDZhakJFUVZGalJGRm5RVVV6ZERSdWR6UmFTSEpDVVRORFlsUm1hMHRCZDB4dWRHOW1MekEwZFdSWGJrZFFTRzhLUW5reU1sTlhkakJOUm01WGJFWndTRE5hZUVwbk5YaFNWRkJ6YW1oR1FuRXJabUpXT1ZOemRrTjJSRlF4TkVGWGVEWlBRMEprUVhkbloxaE5UVUUwUndwQk1WVmtSSGRGUWk5M1VVVkJkMGxJWjBSQlZFSm5UbFpJVTFWRlJFUkJTMEpuWjNKQ1owVkdRbEZqUkVGNlFXWkNaMDVXU0ZOTlJVZEVRVmRuUWxSbUNqQXJibEJXYVZGU2JIWnRiekpQYTI5V1lVeEhUR2hvYTFCNlFuSkNaMDVXU0ZKRlFrRm1PRVZaVkVKbWFHd3hiMlJJVW5kamVtOTJUREprY0dSSGFERUtXV2sxYW1JeU1IWmhTRlp1V2pKc2RWb3lXbWhaTWxWMllUSldlV0p0Vm5OamVURnFZakl4ZEdSWE5YQmtTR3QyVEcxa2NHUkhhREZaYVRrellqTktjZ3BhYlhoMlpETk5kbGx1Vm5CaVIxRjFaVmRHZEdKRlFubGFWMXA2VERKb2JGbFhVbnBNTWpGb1lWYzBkMDlSV1V0TGQxbENRa0ZIUkhaNlFVSkJVVkZ5Q21GSVVqQmpTRTAyVEhrNU1HSXlkR3hpYVRWb1dUTlNjR0l5TlhwTWJXUndaRWRvTVZsdVZucGFXRXBxWWpJMU1GcFhOVEJNYlU1MllsUkJaa0puYjNJS1FtZEZSVUZaVHk5TlFVVkRRa0pHTTJJelNuSmFiWGgyWkRFNWEyRllUbmRaV0ZKcVlVUkJNa0puYjNKQ1owVkZRVmxQTDAxQlJVUkNRMmh0V2xSS2FRcFphbGt5V20xV2JFNXRUWGxaVkdNeldsZE9hbHBxUVRSWlZFRXhUVVJvYTFsNlZUTk9SRWswV2tSUk5FOUhWbXROUWsxSFEybHpSMEZSVVVKbk56aDNDa0ZSVVVWQ1ZVb3hZVmQ0YTAxRGMwZERhWE5IUVZGUlFtYzNPSGRCVVZWRlNGZG9NVm95WkhCaWJXUnRXVmRPYkV3eWRHeGpiVFZzWWtoTmRGa3lPWFFLWWxoV2RXRllValZOUWpCSFEybHpSMEZSVVVKbk56aDNRVkZaUlVRelNteGFiazEyWVVkV2FGcElUWFppVjBad1ltcEJOMEpuYjNKQ1owVkZRVmxQTHdwTlFVVkpRa013VFVzeWFEQmtTRUo2VDJrNGRtUkhPWEphVnpSMVdWZE9NR0ZYT1hWamVUVnVZVmhTYjJSWFNqRmpNbFo1V1RJNWRXUkhWblZrUXpWcUNtSXlNSGRpVVZsTFMzZFpRa0pCUjBSMmVrRkNRMUZTWmtSR01XOWtTRkozWTNwdmRrd3laSEJrUjJneFdXazFhbUl5TUhaaFNGWnVXakpzZFZveVdtZ0tXVEpWZG1FeVZubGliVlp6WTNreGFtSXlNWFJrVnpWd1pFaHJka3h0WkhCa1IyZ3hXV2s1TTJJelNuSmFiWGgyWkROTmRsbHVWbkJpUjFGMVpWZEdkQXBpUlVKNVdsZGFla3d5YUd4WlYxSjZUREl4YUdGWE5IZFBRVmxMUzNkWlFrSkJSMFIyZWtGQ1EyZFJjVVJEYUcxYVZFcHBXV3BaTWxwdFZteE9iVTE1Q2xsVVl6TmFWMDVxV21wQk5GbFVRVEZOUkdocldYcFZNMDVFU1RSYVJGRTBUMGRXYTAxQ2MwZERhWE5IUVZGUlFtYzNPSGRCVVhORlJGRjNUR015Vm5NS1dta3hiMkl6VGpCYVYxRjNVVUZaUzB0M1dVSkNRVWRFZG5wQlFrUkJVWGxFUkVKdlpFaFNkMk42YjNaTU1tUndaRWRvTVZscE5XcGlNakIyWVVoV2JncGFNbXgxV2pKYWFGa3lWWFpoTWxaNVltMVdjMk41TVdwaU1qRjBaRmMxY0dSSWEzZFBRVmxMUzNkWlFrSkJSMFIyZWtGQ1JGRlJjVVJEYUcxYVZFcHBDbGxxV1RKYWJWWnNUbTFOZVZsVVl6TmFWMDVxV21wQk5GbFVRVEZOUkdocldYcFZNMDVFU1RSYVJGRTBUMGRXYTAxQ09FZERhWE5IUVZGUlFtYzNPSGNLUVZFMFJVVlJkMUJqYlZadFkzazViMXBYUm10amVUbDBXVmRzZFUxQ2IwZERhWE5IUVZGUlFtYzNPSGRCVVRoRlJFRjNTMDFVUVROTlZGRXpUbFJWZVFwUFZFRjFRbWR2Y2tKblJVVkJXVTh2VFVGRlVVSkRRVTFJYldnd1pFaENlazlwT0haYU1td3dZVWhXYVV4dFRuWmlVemx2WkZka2JtRlhOVzVhYlVacUNscFVRVmxDWjI5eVFtZEZSVUZaVHk5TlFVVlNRa0Z2VFVORVNURk9la2wzVG5wUmVrMUhNRWREYVhOSFFWRlJRbWMzT0hkQlVrbEZXSGQ0WkdGSVVqQUtZMGhOTmt4NU9XNWhXRkp2WkZkSmRWa3lPWFJNTW1neFdqSmtjR0p0WkcxWlYwNXNUREowYkdOdE5XeGlTRTEwV1RJNWRHSllWblZoV0ZJMVRIazFiZ3BoV0ZKdlpGZEpkbVF5T1hsaE1scHpZak5rZWt3eVNqRmhWM2hyVEc1c2FHSlhlRUZqYlZadFkzazViMXBYUm10amVUbDBXVmRzZFUxRVowZERhWE5IQ2tGUlVVSm5OemgzUVZKTlJVdG5kMjlhYlZWNVdXMUpNazV0V214YVZGcHFUVzFGTTA0eVZtcFpNbGwzVDBkRmQwNVVRVFJhUjAweFRucFJlVTlIVVRBS1QwUm9iRnBFUVdoQ1oyOXlRbWRGUlVGWlR5OU5RVVZWUWtKTlRVVllaSFpqYlhSdFlrYzVNMWd5VW5Cak0wSm9aRWRPYjAxSFVVZERhWE5IUVZGUlFncG5OemgzUVZKVlJWWm5lRlZoU0ZJd1kwaE5Oa3g1T1c1aFdGSnZaRmRKZFZreU9YUk1NbWd4V2pKa2NHSnRaRzFaVjA1c1RESjBiR050Tld4aVNFMTBDbGt5T1hSaVdGWjFZVmhTTlV3eVJtcGtSMngyWW01TmRtTnVWblZqZVRoNlQxUkplazlVVlhoUFJFVjNUa001YUdSSVVteGlXRUl3WTNrNGVFMUNXVWNLUTJselIwRlJVVUpuTnpoM1FWSlpSVU5CZDBkalNGWnBZa2RzYWsxRldVZERhWE5IUVZGUlFtYzNPSGRCVW1kRlQwRjNNbU50Vm5kaWVuQnZaRmRrYmdwaFZ6VnVXbTFHYWxwVE9YSmFXRXAxV2xkNGVreFhUblppVnpFeFltMXNNR1ZVY0hsYVYxazJZMjFXYldONU9XOWFWMFpyWTNrNWRGbFhiSFZOU1VkS0NrSm5iM0pDWjBWRlFXUmFOVUZuVVVOQ1NITkZaVkZDTTBGSVZVRXpWREIzWVhOaVNFVlVTbXBIVWpSamJWZGpNMEZ4U2t0WWNtcGxVRXN6TDJnMGNIa0taME00Y0Rkdk5FRkJRVWRvVm10d1ZYQjNRVUZDUVUxQlVtcENSVUZwUVRGRE1WbHRhMHBtUVVsVVVuQjZlVFZJZEZZNVZrTk5XV2hIYzI5NVpEVm9XZ3BuY0ZkUWNIWm1Wa3RSU1dkUFZrbE1NR2htUWxSWmFqWm5kRFI0YkRGNGNXZFZSVTFwU0RWbWExVm1aek5hY1VaNFJrVkNVMUpCZDBObldVbExiMXBKQ25wcU1FVkJkMDFFWVVGQmQxcFJTWGhCU21FMFVWaHpVRXc0TVV3cmRsaG9SMGRqY2xveGVrVk1XVzVQVldVNFptMVhkWG92UjB4Q1VGUnhMM1k1Y21RS2JsaExTWHA0Y0RGQk5tSkpWMVU0ZW05M1NYZElSMkZSZG01c1JteDFkazVEWnpacFJrbEViRVJJTHk4dlFqVkpMMlFyV2twM2VFRXpUemxQTUhFM2RBcGFaMmRHVWtOVFVqQlVXRFJpTkVzd2NHWmxVUW90TFMwdExVVk9SQ0JEUlZKVVNVWkpRMEZVUlMwdExTMHRDZz09In19fX0="}], "timestampVerificationData":{"rfc3161Timestamps":[{"signedTimestamp":"MIICyzADAgEAMIICwgYJKoZIhvcNAQcCoIICszCCAq8CAQMxDTALBglghkgBZQMEAgEwgbgGCyqGSIb3DQEJEAEEoIGoBIGlMIGiAgEBBgkrBgEEAYO/MAIwMTANBglghkgBZQMEAgEFAAQgnxpe49Bca8BoOZiHasRIYzPfojWVj2O6LRUdho/SpeQCFQCVCrq01TcmtZOV7ywJOzs2MCHCpBgPMjAyNjEwMTkyMjMxMTRaMAMCAQGgMqQwMC4xFTATBgNVBAoTDHNpZ3N0b3JlLmRldjEVMBMGA1UEAxMMc2lnc3RvcmUtdHNhoAAxggHcMIIB2AIBATBRMDkxFTATBgNVBAoTDHNpZ3N0b3JlLmRldjEgMB4GA1UEAxMXc2lnc3RvcmUtdHNhLXNlbGZzaWduZWQCFDoTVC8MkGHuvMFDL8uKjosqI4sMMAsGCWCGSAFlAwQCAaCB/DAaBgkqhkiG9w0BCQMxDQYLKoZIhvcNAQkQAQQwHAYJKoZIhvcNAQkFMQ8XDTI2MTAxOTIyMzExNFowLwYJKoZIhvcNAQkEMSIEILuVLXt8AmHMN6KRJ5J/f2uWMscqD7w9RGP4/WIFhtaqMIGOBgsqhkiG9w0BCRACLzF/MH0wezB5BCCF+Se8B6tiysO0Q1bBDvyBssaIP9p6uebYcNnROs0FtzBVMD2kOzA5MRUwEwYDVQQKEwxzaWdzdG9yZS5kZXYxIDAeBgNVBAMTF3NpZ3N0b3JlLXRzYS1zZWxmc2lnbmVkAhQ6E1QvDJBh7rzBQy/Lio6LKiOLDDAKBggqhkjOPQQDAgRoMGYCMQDBQ5gG1/t9qM5NWnE0tWE0k7iAjLVh1LkWH5yuTH5gMTyC1peLgRopwwY6jQrkyr4CMQCEct/P2loSuprwJPAduGECrNug2LYMwl+jfNReEoXLVUVcTHdo4UN53D3t/T0WJfw="}]}}, "messageSignature":{"messageDigest":{"algorithm":"SHA2_256", "digest":"yL33p7Qd5XvTLNh3pd+D2S1ZbMAfOjbhVRIiHwheZPE="}, "signature":"MEUCIQCzpK+dfRywKWtFUVGT0yEqdp0NvsAEu6PKKYeMnm09qQIgIVcUqEgReIp7mwCL8AlNAhq8Y/xo/MvsOfWdbCvuAYY="}}
build/torch211-rocm72-x86_64-linux/mha.py ADDED
@@ -0,0 +1,332 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FlashAttention forward entry points backed by aiter's Composable-Kernel FMHA.
2
+
3
+ This is the compiled (HIP / Composable Kernel) counterpart to the Triton
4
+ ``aiter-flash-attn`` package. Only the *forward* pass is provided (dense and
5
+ variable-length), including support for learnable attention sinks via
6
+ ``window_size[2]`` / ``s_aux`` (e.g. gpt-oss). The op set ships exclusively
7
+ the CK ``mha_fwd`` / ``mha_varlen_fwd`` kernels, so the sink path is always
8
+ taken (no fall-through to a non-sink ASM kernel).
9
+ """
10
+
11
+ from typing import Optional, Tuple, Union
12
+
13
+ import torch
14
+ from torch import Tensor
15
+
16
+ from ._ops import ops
17
+
18
+
19
+ def _round_multiple(x: int, m: int) -> int:
20
+ return (x + m - 1) // m * m
21
+
22
+
23
+ def _maybe_pad_headdim(t: Tensor, target: int) -> Tensor:
24
+ og = t.size(-1)
25
+ if og == target:
26
+ return t
27
+ return torch.nn.functional.pad(t, [0, target - og])
28
+
29
+
30
+ def _unpack_window(window_size) -> Tuple[int, int, int]:
31
+ left = int(window_size[0])
32
+ right = int(window_size[1])
33
+ sink = int(window_size[2]) if len(window_size) == 3 else 0
34
+ return left, right, sink
35
+
36
+
37
+ def _prep_sink(s_aux: Optional[Tensor], nheads: int, device) -> Optional[Tensor]:
38
+ if s_aux is None:
39
+ return None
40
+ assert s_aux.device == device, "s_aux must be on the same device as q"
41
+ assert s_aux.shape[0] == nheads, "s_aux must have shape [nheads_q]"
42
+ if s_aux.dtype != torch.float32:
43
+ s_aux = s_aux.to(torch.float32)
44
+ return s_aux
45
+
46
+
47
+ def _effective_sink_size(sink_size: int, s_aux: Optional[Tensor]) -> int:
48
+ # The CK kernel only takes the sink path when sink_size > 0. Callers using
49
+ # the transformers attention interface pass the per-head sink logits as
50
+ # ``s_aux`` but leave ``window_size`` a 2-tuple (sink_size == 0), so enable
51
+ # a single learnable sink automatically when s_aux is given.
52
+ if s_aux is not None and sink_size == 0:
53
+ return 1
54
+ return sink_size
55
+
56
+
57
+ def flash_attn_func(
58
+ q: Tensor,
59
+ k: Tensor,
60
+ v: Tensor,
61
+ dropout_p: float = 0.0,
62
+ softmax_scale: Optional[float] = None,
63
+ causal: bool = False,
64
+ window_size: Tuple[int, ...] = (-1, -1, 0),
65
+ bias: Optional[Tensor] = None,
66
+ alibi_slopes: Optional[Tensor] = None,
67
+ return_lse: bool = False,
68
+ return_attn_probs: bool = False,
69
+ s_aux: Optional[Tensor] = None,
70
+ **kwargs,
71
+ ) -> Union[Tensor, Tuple[Tensor, ...]]:
72
+ """Dense FlashAttention forward.
73
+
74
+ ``q, k, v`` have shape ``(batch, seqlen, nheads, headdim)``. ``window_size``
75
+ is ``(left, right, sink_size)``; ``s_aux`` is an optional ``(nheads_q,)``
76
+ fp32 tensor of learnable sink logits. Extra keyword arguments (e.g. those
77
+ passed by the transformers flash-attention interface) are ignored.
78
+ """
79
+ if softmax_scale is None:
80
+ softmax_scale = q.shape[-1] ** (-0.5)
81
+ window_size_left, window_size_right, sink_size = _unpack_window(window_size)
82
+
83
+ nheads_q = q.size(2)
84
+ s_aux = _prep_sink(s_aux, nheads_q, q.device)
85
+ sink_size = _effective_sink_size(sink_size, s_aux)
86
+
87
+ head_q_og = q.size(-1)
88
+ head_v_og = v.size(-1)
89
+ head_q = _round_multiple(head_q_og, 8)
90
+ head_v = _round_multiple(head_v_og, 8)
91
+ q = _maybe_pad_headdim(q, head_q)
92
+ k = _maybe_pad_headdim(k, head_q)
93
+ v = _maybe_pad_headdim(v, head_v)
94
+
95
+ return_softmax = return_attn_probs and dropout_p > 0
96
+ out, softmax_lse, S_dmask, _ = ops.mha_fwd(
97
+ q,
98
+ k,
99
+ v,
100
+ dropout_p,
101
+ softmax_scale,
102
+ causal,
103
+ window_size_left,
104
+ window_size_right,
105
+ sink_size,
106
+ return_lse or return_attn_probs,
107
+ return_softmax,
108
+ None, # cu_seqlens_q
109
+ None, # cu_seqlens_kv
110
+ None, # out
111
+ bias,
112
+ alibi_slopes,
113
+ None, # q_descale
114
+ None, # k_descale
115
+ None, # v_descale
116
+ s_aux,
117
+ None, # gen
118
+ )
119
+
120
+ out = out[..., :head_v_og]
121
+ if not (return_lse or return_attn_probs):
122
+ return out
123
+ result = (out,)
124
+ if return_lse:
125
+ result = result + (softmax_lse,)
126
+ if return_attn_probs:
127
+ result = result + (S_dmask,)
128
+ return result
129
+
130
+
131
+ def flash_attn_varlen_func(
132
+ q: Tensor,
133
+ k: Tensor,
134
+ v: Tensor,
135
+ cu_seqlens_q: Tensor,
136
+ cu_seqlens_k: Tensor,
137
+ max_seqlen_q: int,
138
+ max_seqlen_k: int,
139
+ dropout_p: float = 0.0,
140
+ softmax_scale: Optional[float] = None,
141
+ causal: bool = False,
142
+ window_size: Tuple[int, ...] = (-1, -1, 0),
143
+ bias: Optional[Tensor] = None,
144
+ alibi_slopes: Optional[Tensor] = None,
145
+ return_lse: bool = False,
146
+ return_attn_probs: bool = False,
147
+ block_table: Optional[Tensor] = None,
148
+ min_seqlen_q: int = 0,
149
+ logits_soft_cap: float = 0.0,
150
+ zero_tensors: bool = False,
151
+ cu_seqlens_q_padded: Optional[Tensor] = None,
152
+ cu_seqlens_k_padded: Optional[Tensor] = None,
153
+ s_aux: Optional[Tensor] = None,
154
+ **kwargs,
155
+ ) -> Union[Tensor, Tuple[Tensor, ...]]:
156
+ """Variable-length / packed FlashAttention forward.
157
+
158
+ ``q`` has shape ``(total_q, nheads, headdim)``; ``cu_seqlens_*`` are int32
159
+ cumulative offsets of shape ``(batch + 1,)``. Sinks are supported through
160
+ ``window_size[2]`` / ``s_aux`` as in :func:`flash_attn_func`. Extra keyword
161
+ arguments (e.g. from the transformers flash-attention interface) are ignored.
162
+ """
163
+ if softmax_scale is None:
164
+ softmax_scale = q.shape[-1] ** (-0.5)
165
+ window_size_left, window_size_right, sink_size = _unpack_window(window_size)
166
+
167
+ nheads_q = q.size(1)
168
+ s_aux = _prep_sink(s_aux, nheads_q, q.device)
169
+ sink_size = _effective_sink_size(sink_size, s_aux)
170
+
171
+ head_q_og = q.size(-1)
172
+ head_v_og = v.size(-1)
173
+ head_q = _round_multiple(head_q_og, 8)
174
+ head_v = _round_multiple(head_v_og, 8)
175
+ q = _maybe_pad_headdim(q, head_q)
176
+ k = _maybe_pad_headdim(k, head_q)
177
+ v = _maybe_pad_headdim(v, head_v)
178
+
179
+ return_softmax = return_attn_probs and dropout_p > 0
180
+ out, softmax_lse, S_dmask, _ = ops.mha_varlen_fwd(
181
+ q,
182
+ k,
183
+ v,
184
+ cu_seqlens_q,
185
+ cu_seqlens_k,
186
+ max_seqlen_q,
187
+ max_seqlen_k,
188
+ min_seqlen_q,
189
+ dropout_p,
190
+ softmax_scale,
191
+ logits_soft_cap,
192
+ zero_tensors,
193
+ causal,
194
+ window_size_left,
195
+ window_size_right,
196
+ sink_size,
197
+ return_lse or return_attn_probs,
198
+ return_softmax,
199
+ None, # out
200
+ block_table,
201
+ bias,
202
+ alibi_slopes,
203
+ None, # q_descale
204
+ None, # k_descale
205
+ None, # v_descale
206
+ None, # gen
207
+ cu_seqlens_q_padded,
208
+ cu_seqlens_k_padded,
209
+ s_aux,
210
+ )
211
+
212
+ out = out[..., :head_v_og]
213
+ if not (return_lse or return_attn_probs):
214
+ return out
215
+ result = (out,)
216
+ if return_lse:
217
+ result = result + (softmax_lse,)
218
+ if return_attn_probs:
219
+ result = result + (S_dmask,)
220
+ return result
221
+
222
+
223
+ def _append_to_contiguous_cache(k_cache, v_cache, k, v, cache_seqlens, Sq):
224
+ """In-place append of new k/v into a contiguous cache at per-batch offsets."""
225
+ B = k_cache.shape[0]
226
+ if isinstance(cache_seqlens, int):
227
+ cache_seqlens = torch.full((B,), cache_seqlens, device=k_cache.device, dtype=torch.long)
228
+ pos = cache_seqlens.view(B, 1).to(torch.long) + torch.arange(Sq, device=k_cache.device).view(1, Sq)
229
+ bidx = torch.arange(B, device=k_cache.device).view(B, 1).expand(B, Sq)
230
+ k_cache[bidx, pos] = k.to(k_cache.dtype)
231
+ v_cache[bidx, pos] = v.to(v_cache.dtype)
232
+
233
+
234
+ def flash_attn_with_kvcache(
235
+ q: Tensor,
236
+ k_cache: Tensor,
237
+ v_cache: Tensor,
238
+ k: Optional[Tensor] = None,
239
+ v: Optional[Tensor] = None,
240
+ rotary_cos: Optional[Tensor] = None,
241
+ rotary_sin: Optional[Tensor] = None,
242
+ cache_seqlens=None,
243
+ block_table: Optional[Tensor] = None,
244
+ softmax_scale: Optional[float] = None,
245
+ causal: bool = True,
246
+ window_size: Tuple[int, ...] = (-1, -1, 0),
247
+ s_aux: Optional[Tensor] = None,
248
+ return_lse: bool = False,
249
+ **kwargs,
250
+ ) -> Union[Tensor, Tuple[Tensor, ...]]:
251
+ """FlashAttention against a KV cache, built on the CK split-KV kernel.
252
+
253
+ ``q`` is ``(batch, seqlen_q, nheads, headdim)``. For a contiguous cache,
254
+ ``k_cache``/``v_cache`` are ``(batch, seqlen_cache, nheads_k, headdim)`` and
255
+ ``cache_seqlens`` (int or ``(batch,)``) is the number of valid cache entries
256
+ *before* appending. If ``k``/``v`` (the new tokens) are given, they are
257
+ written into the cache in place. The query attends to the valid prefix
258
+ (``cache_seqlens`` + ``seqlen_q``) with bottom-right causal alignment.
259
+
260
+ Notes / current limitations:
261
+ - Rotary embedding is not applied here; pass ``q``/``k`` already rotated
262
+ (``rotary_cos``/``rotary_sin`` are rejected).
263
+ - Paged caches are addressed via ``block_table`` (forwarded to the split-KV
264
+ varlen kernel); in-cache append for paged layouts must be done by the
265
+ caller (pass ``k=v=None``).
266
+ """
267
+ if rotary_cos is not None or rotary_sin is not None:
268
+ raise NotImplementedError(
269
+ "flash_attn_with_kvcache does not apply rotary; rotate q/k beforehand."
270
+ )
271
+ if softmax_scale is None:
272
+ softmax_scale = q.shape[-1] ** (-0.5)
273
+ window_size_left, window_size_right, sink_size = _unpack_window(window_size)
274
+ B, Sq, Hq, D = q.shape
275
+ s_aux = _prep_sink(s_aux, Hq, q.device)
276
+ sink_size = _effective_sink_size(sink_size, s_aux)
277
+
278
+ paged = block_table is not None
279
+
280
+ # ---- append new tokens into the cache (contiguous layout only) ----
281
+ if k is not None and not paged:
282
+ assert cache_seqlens is not None, "cache_seqlens required to append into the cache"
283
+ _append_to_contiguous_cache(k_cache, v_cache, k, v, cache_seqlens, Sq)
284
+ elif k is not None and paged:
285
+ raise NotImplementedError(
286
+ "in-cache append for paged layouts is not supported; append externally "
287
+ "and call with k=v=None."
288
+ )
289
+
290
+ appended = 0 if k is None else Sq
291
+
292
+ # ---- key lengths ----
293
+ if cache_seqlens is None:
294
+ klens = torch.full((B,), k_cache.shape[1], device=q.device, dtype=torch.long)
295
+ elif isinstance(cache_seqlens, int):
296
+ klens = torch.full((B,), cache_seqlens + appended, device=q.device, dtype=torch.long)
297
+ else:
298
+ klens = cache_seqlens.to(q.device, torch.long) + appended
299
+
300
+ uniform = bool((klens == klens[0]).all().item())
301
+
302
+ # ---- fast path: contiguous cache, uniform length -> dense kernel ----
303
+ if not paged and uniform:
304
+ L = int(klens[0].item())
305
+ return flash_attn_func(
306
+ q, k_cache[:, :L], v_cache[:, :L],
307
+ softmax_scale=softmax_scale, causal=causal,
308
+ window_size=(window_size_left, window_size_right, sink_size),
309
+ s_aux=s_aux, return_lse=return_lse,
310
+ )
311
+
312
+ # ---- general path: variable cache lengths -> split-KV varlen kernel ----
313
+ cu_q = torch.arange(0, (B + 1) * Sq, Sq, device=q.device, dtype=torch.int32)
314
+ cu_k = torch.zeros(B + 1, device=q.device, dtype=torch.int32)
315
+ cu_k[1:] = torch.cumsum(klens, 0).to(torch.int32)
316
+ q_packed = q.reshape(B * Sq, Hq, D)
317
+ if paged:
318
+ k_packed, v_packed = k_cache, v_cache # kernel gathers via block_table
319
+ else:
320
+ k_packed = torch.cat([k_cache[b, : int(klens[b])] for b in range(B)], dim=0)
321
+ v_packed = torch.cat([v_cache[b, : int(klens[b])] for b in range(B)], dim=0)
322
+
323
+ out = flash_attn_varlen_func(
324
+ q_packed, k_packed, v_packed, cu_q, cu_k, Sq, int(klens.max().item()),
325
+ softmax_scale=softmax_scale, causal=causal,
326
+ window_size=(window_size_left, window_size_right, sink_size),
327
+ block_table=block_table, s_aux=s_aux, return_lse=return_lse,
328
+ )
329
+ if return_lse:
330
+ out, lse = out
331
+ return out.reshape(B, Sq, Hq, D), lse
332
+ return out.reshape(B, Sq, Hq, D)
build/torch211-rocm72-x86_64-linux/symbols.json ADDED
@@ -0,0 +1,349 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "schema_version": 1,
3
+ "module": "_aiter_flash_attn_ck_rocm_fe2bb66",
4
+ "functions": [
5
+ {
6
+ "name": "flash_attn_func",
7
+ "qualname": "flash_attn_func",
8
+ "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha",
9
+ "kind": "function",
10
+ "doc": "Dense FlashAttention forward.\n\n``q, k, v`` have shape ``(batch, seqlen, nheads, headdim)``. ``window_size``\nis ``(left, right, sink_size)``; ``s_aux`` is an optional ``(nheads_q,)``\nfp32 tensor of learnable sink logits. Extra keyword arguments (e.g. those\npassed by the transformers flash-attention interface) are ignored.",
11
+ "signature": {
12
+ "parameters": [
13
+ {
14
+ "name": "q",
15
+ "kind": "POSITIONAL_OR_KEYWORD",
16
+ "default": null,
17
+ "annotation": "torch.Tensor"
18
+ },
19
+ {
20
+ "name": "k",
21
+ "kind": "POSITIONAL_OR_KEYWORD",
22
+ "default": null,
23
+ "annotation": "torch.Tensor"
24
+ },
25
+ {
26
+ "name": "v",
27
+ "kind": "POSITIONAL_OR_KEYWORD",
28
+ "default": null,
29
+ "annotation": "torch.Tensor"
30
+ },
31
+ {
32
+ "name": "dropout_p",
33
+ "kind": "POSITIONAL_OR_KEYWORD",
34
+ "default": "0.0",
35
+ "annotation": "float"
36
+ },
37
+ {
38
+ "name": "softmax_scale",
39
+ "kind": "POSITIONAL_OR_KEYWORD",
40
+ "default": "None",
41
+ "annotation": "float | None"
42
+ },
43
+ {
44
+ "name": "causal",
45
+ "kind": "POSITIONAL_OR_KEYWORD",
46
+ "default": "False",
47
+ "annotation": "bool"
48
+ },
49
+ {
50
+ "name": "window_size",
51
+ "kind": "POSITIONAL_OR_KEYWORD",
52
+ "default": "(-1, -1, 0)",
53
+ "annotation": "Tuple[int, ...]"
54
+ },
55
+ {
56
+ "name": "bias",
57
+ "kind": "POSITIONAL_OR_KEYWORD",
58
+ "default": "None",
59
+ "annotation": "torch.Tensor | None"
60
+ },
61
+ {
62
+ "name": "alibi_slopes",
63
+ "kind": "POSITIONAL_OR_KEYWORD",
64
+ "default": "None",
65
+ "annotation": "torch.Tensor | None"
66
+ },
67
+ {
68
+ "name": "return_lse",
69
+ "kind": "POSITIONAL_OR_KEYWORD",
70
+ "default": "False",
71
+ "annotation": "bool"
72
+ },
73
+ {
74
+ "name": "return_attn_probs",
75
+ "kind": "POSITIONAL_OR_KEYWORD",
76
+ "default": "False",
77
+ "annotation": "bool"
78
+ },
79
+ {
80
+ "name": "s_aux",
81
+ "kind": "POSITIONAL_OR_KEYWORD",
82
+ "default": "None",
83
+ "annotation": "torch.Tensor | None"
84
+ },
85
+ {
86
+ "name": "kwargs",
87
+ "kind": "VAR_KEYWORD",
88
+ "default": null,
89
+ "annotation": null
90
+ }
91
+ ],
92
+ "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]"
93
+ }
94
+ },
95
+ {
96
+ "name": "flash_attn_varlen_func",
97
+ "qualname": "flash_attn_varlen_func",
98
+ "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha",
99
+ "kind": "function",
100
+ "doc": "Variable-length / packed FlashAttention forward.\n\n``q`` has shape ``(total_q, nheads, headdim)``; ``cu_seqlens_*`` are int32\ncumulative offsets of shape ``(batch + 1,)``. Sinks are supported through\n``window_size[2]`` / ``s_aux`` as in :func:`flash_attn_func`. Extra keyword\narguments (e.g. from the transformers flash-attention interface) are ignored.",
101
+ "signature": {
102
+ "parameters": [
103
+ {
104
+ "name": "q",
105
+ "kind": "POSITIONAL_OR_KEYWORD",
106
+ "default": null,
107
+ "annotation": "torch.Tensor"
108
+ },
109
+ {
110
+ "name": "k",
111
+ "kind": "POSITIONAL_OR_KEYWORD",
112
+ "default": null,
113
+ "annotation": "torch.Tensor"
114
+ },
115
+ {
116
+ "name": "v",
117
+ "kind": "POSITIONAL_OR_KEYWORD",
118
+ "default": null,
119
+ "annotation": "torch.Tensor"
120
+ },
121
+ {
122
+ "name": "cu_seqlens_q",
123
+ "kind": "POSITIONAL_OR_KEYWORD",
124
+ "default": null,
125
+ "annotation": "torch.Tensor"
126
+ },
127
+ {
128
+ "name": "cu_seqlens_k",
129
+ "kind": "POSITIONAL_OR_KEYWORD",
130
+ "default": null,
131
+ "annotation": "torch.Tensor"
132
+ },
133
+ {
134
+ "name": "max_seqlen_q",
135
+ "kind": "POSITIONAL_OR_KEYWORD",
136
+ "default": null,
137
+ "annotation": "int"
138
+ },
139
+ {
140
+ "name": "max_seqlen_k",
141
+ "kind": "POSITIONAL_OR_KEYWORD",
142
+ "default": null,
143
+ "annotation": "int"
144
+ },
145
+ {
146
+ "name": "dropout_p",
147
+ "kind": "POSITIONAL_OR_KEYWORD",
148
+ "default": "0.0",
149
+ "annotation": "float"
150
+ },
151
+ {
152
+ "name": "softmax_scale",
153
+ "kind": "POSITIONAL_OR_KEYWORD",
154
+ "default": "None",
155
+ "annotation": "float | None"
156
+ },
157
+ {
158
+ "name": "causal",
159
+ "kind": "POSITIONAL_OR_KEYWORD",
160
+ "default": "False",
161
+ "annotation": "bool"
162
+ },
163
+ {
164
+ "name": "window_size",
165
+ "kind": "POSITIONAL_OR_KEYWORD",
166
+ "default": "(-1, -1, 0)",
167
+ "annotation": "Tuple[int, ...]"
168
+ },
169
+ {
170
+ "name": "bias",
171
+ "kind": "POSITIONAL_OR_KEYWORD",
172
+ "default": "None",
173
+ "annotation": "torch.Tensor | None"
174
+ },
175
+ {
176
+ "name": "alibi_slopes",
177
+ "kind": "POSITIONAL_OR_KEYWORD",
178
+ "default": "None",
179
+ "annotation": "torch.Tensor | None"
180
+ },
181
+ {
182
+ "name": "return_lse",
183
+ "kind": "POSITIONAL_OR_KEYWORD",
184
+ "default": "False",
185
+ "annotation": "bool"
186
+ },
187
+ {
188
+ "name": "return_attn_probs",
189
+ "kind": "POSITIONAL_OR_KEYWORD",
190
+ "default": "False",
191
+ "annotation": "bool"
192
+ },
193
+ {
194
+ "name": "block_table",
195
+ "kind": "POSITIONAL_OR_KEYWORD",
196
+ "default": "None",
197
+ "annotation": "torch.Tensor | None"
198
+ },
199
+ {
200
+ "name": "min_seqlen_q",
201
+ "kind": "POSITIONAL_OR_KEYWORD",
202
+ "default": "0",
203
+ "annotation": "int"
204
+ },
205
+ {
206
+ "name": "logits_soft_cap",
207
+ "kind": "POSITIONAL_OR_KEYWORD",
208
+ "default": "0.0",
209
+ "annotation": "float"
210
+ },
211
+ {
212
+ "name": "zero_tensors",
213
+ "kind": "POSITIONAL_OR_KEYWORD",
214
+ "default": "False",
215
+ "annotation": "bool"
216
+ },
217
+ {
218
+ "name": "cu_seqlens_q_padded",
219
+ "kind": "POSITIONAL_OR_KEYWORD",
220
+ "default": "None",
221
+ "annotation": "torch.Tensor | None"
222
+ },
223
+ {
224
+ "name": "cu_seqlens_k_padded",
225
+ "kind": "POSITIONAL_OR_KEYWORD",
226
+ "default": "None",
227
+ "annotation": "torch.Tensor | None"
228
+ },
229
+ {
230
+ "name": "s_aux",
231
+ "kind": "POSITIONAL_OR_KEYWORD",
232
+ "default": "None",
233
+ "annotation": "torch.Tensor | None"
234
+ },
235
+ {
236
+ "name": "kwargs",
237
+ "kind": "VAR_KEYWORD",
238
+ "default": null,
239
+ "annotation": null
240
+ }
241
+ ],
242
+ "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]"
243
+ }
244
+ },
245
+ {
246
+ "name": "flash_attn_with_kvcache",
247
+ "qualname": "flash_attn_with_kvcache",
248
+ "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha",
249
+ "kind": "function",
250
+ "doc": "FlashAttention against a KV cache, built on the CK split-KV kernel.\n\n``q`` is ``(batch, seqlen_q, nheads, headdim)``. For a contiguous cache,\n``k_cache``/``v_cache`` are ``(batch, seqlen_cache, nheads_k, headdim)`` and\n``cache_seqlens`` (int or ``(batch,)``) is the number of valid cache entries\n*before* appending. If ``k``/``v`` (the new tokens) are given, they are\nwritten into the cache in place. The query attends to the valid prefix\n(``cache_seqlens`` + ``seqlen_q``) with bottom-right causal alignment.\n\nNotes / current limitations:\n- Rotary embedding is not applied here; pass ``q``/``k`` already rotated\n (``rotary_cos``/``rotary_sin`` are rejected).\n- Paged caches are addressed via ``block_table`` (forwarded to the split-KV\n varlen kernel); in-cache append for paged layouts must be done by the\n caller (pass ``k=v=None``).",
251
+ "signature": {
252
+ "parameters": [
253
+ {
254
+ "name": "q",
255
+ "kind": "POSITIONAL_OR_KEYWORD",
256
+ "default": null,
257
+ "annotation": "torch.Tensor"
258
+ },
259
+ {
260
+ "name": "k_cache",
261
+ "kind": "POSITIONAL_OR_KEYWORD",
262
+ "default": null,
263
+ "annotation": "torch.Tensor"
264
+ },
265
+ {
266
+ "name": "v_cache",
267
+ "kind": "POSITIONAL_OR_KEYWORD",
268
+ "default": null,
269
+ "annotation": "torch.Tensor"
270
+ },
271
+ {
272
+ "name": "k",
273
+ "kind": "POSITIONAL_OR_KEYWORD",
274
+ "default": "None",
275
+ "annotation": "torch.Tensor | None"
276
+ },
277
+ {
278
+ "name": "v",
279
+ "kind": "POSITIONAL_OR_KEYWORD",
280
+ "default": "None",
281
+ "annotation": "torch.Tensor | None"
282
+ },
283
+ {
284
+ "name": "rotary_cos",
285
+ "kind": "POSITIONAL_OR_KEYWORD",
286
+ "default": "None",
287
+ "annotation": "torch.Tensor | None"
288
+ },
289
+ {
290
+ "name": "rotary_sin",
291
+ "kind": "POSITIONAL_OR_KEYWORD",
292
+ "default": "None",
293
+ "annotation": "torch.Tensor | None"
294
+ },
295
+ {
296
+ "name": "cache_seqlens",
297
+ "kind": "POSITIONAL_OR_KEYWORD",
298
+ "default": "None",
299
+ "annotation": null
300
+ },
301
+ {
302
+ "name": "block_table",
303
+ "kind": "POSITIONAL_OR_KEYWORD",
304
+ "default": "None",
305
+ "annotation": "torch.Tensor | None"
306
+ },
307
+ {
308
+ "name": "softmax_scale",
309
+ "kind": "POSITIONAL_OR_KEYWORD",
310
+ "default": "None",
311
+ "annotation": "float | None"
312
+ },
313
+ {
314
+ "name": "causal",
315
+ "kind": "POSITIONAL_OR_KEYWORD",
316
+ "default": "True",
317
+ "annotation": "bool"
318
+ },
319
+ {
320
+ "name": "window_size",
321
+ "kind": "POSITIONAL_OR_KEYWORD",
322
+ "default": "(-1, -1, 0)",
323
+ "annotation": "Tuple[int, ...]"
324
+ },
325
+ {
326
+ "name": "s_aux",
327
+ "kind": "POSITIONAL_OR_KEYWORD",
328
+ "default": "None",
329
+ "annotation": "torch.Tensor | None"
330
+ },
331
+ {
332
+ "name": "return_lse",
333
+ "kind": "POSITIONAL_OR_KEYWORD",
334
+ "default": "False",
335
+ "annotation": "bool"
336
+ },
337
+ {
338
+ "name": "kwargs",
339
+ "kind": "VAR_KEYWORD",
340
+ "default": null,
341
+ "annotation": null
342
+ }
343
+ ],
344
+ "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]"
345
+ }
346
+ }
347
+ ],
348
+ "layers": []
349
+ }
build/torch212-rocm71-x86_64-linux/__init__.py ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """AITER Composable-Kernel FlashAttention kernels for AMD ROCm (compiled HIP).
2
+
3
+ Compiled (Composable Kernel / HIP) counterpart to the Triton-based
4
+ ``aiter-flash-attn`` package, repackaged from the CK FMHA forward path of the
5
+ `ROCm/aiter <https://github.com/ROCm/aiter>`_ project as a self-contained
6
+ Hugging Face Hub kernel. Exposes the FlashAttention forward entry points with
7
+ learnable attention sink support (e.g. gpt-oss).
8
+ """
9
+
10
+ from .mha import (
11
+ flash_attn_func,
12
+ flash_attn_varlen_func,
13
+ flash_attn_with_kvcache,
14
+ )
15
+
16
+
17
+ __kernel_metadata__ = {
18
+ "license": "mit",
19
+ }
20
+
21
+
22
+ __all__ = [
23
+ "__kernel_metadata__",
24
+ "flash_attn_func",
25
+ "flash_attn_varlen_func",
26
+ "flash_attn_with_kvcache",
27
+ ]
build/torch212-rocm71-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:78095dfb6de1a2d229d40cdf34e125605bb33d7fa467134d236cb51df66eaf9b
3
+ size 201962680
build/torch212-rocm71-x86_64-linux/_ops.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _aiter_flash_attn_ck_rocm_fe2bb66
3
+ ops = torch.ops._aiter_flash_attn_ck_rocm_fe2bb66
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ """
7
+ Prefix op by namespace.
8
+ """
9
+ return f"_aiter_flash_attn_ck_rocm_fe2bb66::{op_name}"
build/torch212-rocm71-x86_64-linux/metadata.json ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "aiter-flash-attn-ck",
3
+ "id": "_aiter_flash_attn_ck_rocm_fe2bb66",
4
+ "version": 1,
5
+ "kernels-minver": "0.14.0",
6
+ "license": "MIT",
7
+ "python-depends": [],
8
+ "kernel-depends": [],
9
+ "backend": {
10
+ "type": "rocm",
11
+ "archs": [
12
+ "gfx942",
13
+ "gfx950"
14
+ ]
15
+ },
16
+ "digest": {
17
+ "algorithm": "sha256",
18
+ "files": {
19
+ "__init__.py": "7wj1GTm2bqTLLCyT4xLuAk/+JWYrhINjY/ODMNWhgZ0=",
20
+ "_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so": "eAld+23hotIp1AzfNOElYFuzPX+kZxNNI2y1HfZur5s=",
21
+ "_ops.py": "4WkdYDWWJp60zep+oW/5ym9vIeVMNfrgRCkP3o/w7yU=",
22
+ "mha.py": "zCAjY4O+DiCzmEL+W9sqXxZaqjsj4RJbWFcyPpUZejk=",
23
+ "symbols.json": "rhnF4H8Jsb6I+YPxIVFMgnLcTQJODEXbZ7ge77NZXAU="
24
+ }
25
+ },
26
+ "provenance": {
27
+ "kernel-builder": {
28
+ "version": "0.18.0-dev0",
29
+ "commit": "d054906a13710b811b96697b4635f03020842733",
30
+ "dirty": false
31
+ },
32
+ "kernel": {
33
+ "commit": "fe2bb66fee6c2a77eccf08a0508dc57428d488ed",
34
+ "dirty": false
35
+ }
36
+ }
37
+ }
build/torch212-rocm71-x86_64-linux/metadata.json.sigstore ADDED
@@ -0,0 +1 @@
 
 
1
+ {"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json", "verificationMaterial":{"certificate":{"rawBytes":"MIIHKzCCBrKgAwIBAgIUNCpKwqkj9HF7/3kzrniINKWzzUMwCgYIKoZIzj0EAwMwNzEVMBMGA1UEChMMc2lnc3RvcmUuZGV2MR4wHAYDVQQDExVzaWdzdG9yZS1pbnRlcm1lZGlhdGUwHhcNMjYxMDE5MjIzMTE1WhcNMjYxMDE5MjI0MTE1WjAAMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE3edsuwnJ7jqqemLx4AZlgRX5HxhcWRE6INRFuuHbmZJClfx/ipjms6nRAaXEXcRASW3lKsIpYeSr/T1KEb00/6OCBdEwggXNMA4GA1UdDwEB/wQEAwIHgDATBgNVHSUEDDAKBggrBgEFBQcDAzAfBgNVHSMEGDAWgBTf0+nPViQRlvmo2OkoVaLGLhhkPzBrBgNVHREBAf8EYTBfhl1odHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkvLmdpdGh1Yi93b3JrZmxvd3MvYnVpbGQueWFtbEByZWZzL2hlYWRzL21haW4wOQYKKwYBBAGDvzABAQQraHR0cHM6Ly90b2tlbi5hY3Rpb25zLmdpdGh1YnVzZXJjb250ZW50LmNvbTAfBgorBgEEAYO/MAECBBF3b3JrZmxvd19kaXNwYXRjaDA2BgorBgEEAYO/MAEDBChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMBMGCisGAQQBg78wAQQEBUJ1aWxkMCsGCisGAQQBg78wAQUEHWh1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5MB0GCisGAQQBg78wAQYED3JlZnMvaGVhZHMvbWFpbjA7BgorBgEEAYO/MAEIBC0MK2h0dHBzOi8vdG9rZW4uYWN0aW9ucy5naXRodWJ1c2VyY29udGVudC5jb20wbQYKKwYBBAGDvzABCQRfDF1odHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkvLmdpdGh1Yi93b3JrZmxvd3MvYnVpbGQueWFtbEByZWZzL2hlYWRzL21haW4wOAYKKwYBBAGDvzABCgQqDChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMBsGCisGAQQBg78wAQsEDQwLc2VsZi1ob3N0ZWQwQAYKKwYBBAGDvzABDAQyDDBodHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkwOAYKKwYBBAGDvzABDQQqDChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMB8GCisGAQQBg78wAQ4EEQwPcmVmcy9oZWFkcy9tYWluMBoGCisGAQQBg78wAQ8EDAwKMTA3MTQ3NTUyOTAuBgorBgEEAYO/MAEQBCAMHmh0dHBzOi8vZ2l0aHViLmNvbS9odWdnaW5nZmFjZTAYBgorBgEEAYO/MAERBAoMCDI1NzIwNzQzMG0GCisGAQQBg78wARIEXwxdaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5Ly5naXRodWIvd29ya2Zsb3dzL2J1aWxkLnlhbWxAcmVmcy9oZWFkcy9tYWluMDgGCisGAQQBg78wARMEKgwoZmUyYmI2NmZlZTZjMmE3N2VjY2YwOGEwNTA4ZGM1NzQyOGQ0ODhlZDAhBgorBgEEAYO/MAEUBBMMEXdvcmtmbG93X2Rpc3BhdGNoMGQGCisGAQQBg78wARUEVgxUaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5L2FjdGlvbnMvcnVucy8zOTIzOTUxODEwNC9hdHRlbXB0cy8xMBYGCisGAQQBg78wARYECAwGcHVibGljMEYGCisGAQQBg78wARgEOAw2cmVwbzpodWdnaW5nZmFjZS9rZXJuZWxzLWNvbW11bml0eTpyZWY6cmVmcy9oZWFkcy9tYWluMIGKBgorBgEEAdZ5AgQCBHwEegB4AHYA3T0wasbHETJjGR4cmWc3AqJKXrjePK3/h4pygC8p7o4AAAGhVkpa6AAABAMARzBFAiEApQwCCX96bimcS6xRJKWJBsUiSoDvd+Pk/QYYFcktSVkCIEJ9/2rGZ8klXslytVkeKFQzr+e4l7NootT2Du8uVhacMAoGCCqGSM49BAMDA2cAMGQCMBAEOmqdi0D8R2V1qMg4/OyrvTJWu4oal0KEefWlu6GF7soePSHkA07ZyEiste4BDgIwf5dWNp7viPgDNSfubu6HszrI/4FgaMp5hPAHf5f//UA5nMbgENnYVvxWFKBCVZ8j"}, "tlogEntries":[{"logIndex":"3327211868", "logId":{"keyId":"wNI9atQGlz+VWfO6LRygH4QUfY/8W4RFwiT5i5WRgB0="}, "kindVersion":{"kind":"hashedrekord", "version":"0.0.1"}, "integratedTime":"1792449076", "inclusionPromise":{"signedEntryTimestamp":"MEUCIDMDUHqwsYRzaWixuHGJjWHFJ4/iRt+2UmGi39CfIy1dAiEA+wYw2O5mzlJ9y2j5cF2T4wEuFqjVZysBRi7ItBwq/98="}, "inclusionProof":{"logIndex":"3205307606", "rootHash":"0n+xNJnLMgp7sozEH6SRp+aDmxlicUgusBhZlZgsqJ0=", "treeSize":"3205307650", "hashes":["tfCB68vh29EGCuUdCAh0c7SkWTkhxdx4BgzG97ksQ6c=", "7mk/WY2lqGuRTbQKWtV6zQnOdC4m3It2CB/bnMVG0T0=", "cCW6sHzYYMHsBtt/nC0rcoi+UvaxxDmIhd7a2M452G4=", "dtWd+u22D+tZsDa84T5ZJW4w8w8UVj9lD0NJJDqjoHc=", "63fJ5uSg1REGSodPGe4zXFCUfdneQIGhUrpJayO0qkA=", "bFh/+uIRRy9QjkgdrzkEoxjpRf5jf0Ls3F1ULwcdbR4=", "M6d/KuQ//DRltSQtXzLZ1oBzSuFsf9G7CmHdK66xTD8=", "u+n9nq8zizYghdQXamNSsJ1yejTkpNxZoMaQxLGJgr8=", "w3goxeMWPeeGsKDqCTPfJBoJKCBga4F5WnerVtyidt8=", "0aSE/O6jCq82vSwoybvpblRnnP72+1gEInKy6Wyc3E0=", "5egUlZ4Pi0Ix/qLyCDXocsWywJCHqrde/S06vOZvlOM=", "QsXufUpoKtSjKAB19K1G00umQpWvPHlBNuRGFiiVjQQ=", "QIMkwQbwAewhLJd//5FFwI3SnyNQBGjCgJrelfG8vhc=", "Et17CACUv9QN64irUJuBjsrgRGkGnnJ9xacRV/aBdwQ=", "lqgEwkxhDMs/gdIAE4lmjfMwg7nDmv7lda6O0LGliCw=", "HovTPBWzoErvUg1u5sIi3x1ZonZ7yJekVsBrmpomCIA=", "G+nk7mgE5h6dUmv2UY7NDJrY3WvrZY31raaO1AETt+o=", "qlTWNDzERnTBQVcHHXXVPjiUP0SsikGxfcxVlgqVFDo=", "gg9GNApNke3+zNhl4tsWtmIycCFrOMCB3lys6XuSQNQ=", "NlOclldmKPA1tMKIskqHAwuQDYHp31YwpK+OOMrMccc=", "qxzHanAzz57SDdmJe0B7bJK72NTIBbwEMGBBKvDOROw=", "xH/DCseLHr9eKoYT8qsORZK7zVdEGYWHuVtsVrD95wY="], "checkpoint":{"envelope":"rekor.sigstore.dev - 1193050959916656506\n3205307650\n0n+xNJnLMgp7sozEH6SRp+aDmxlicUgusBhZlZgsqJ0=\n\n— rekor.sigstore.dev wNI9ajBFAiEA8G/cygCtz/9naiuZybe4zJFaOz4VLpZ1iK0GcNhfBekCIHsqXQhL6AgKCM9O4yJHjKODCbwaSWuacB9SlITFqmLl\n"}}, "canonicalizedBody":"eyJhcGlWZXJzaW9uIjoiMC4wLjEiLCJraW5kIjoiaGFzaGVkcmVrb3JkIiwic3BlYyI6eyJkYXRhIjp7Imhhc2giOnsiYWxnb3JpdGhtIjoic2hhMjU2IiwidmFsdWUiOiI0MDgzOGUyOGE4MjJkMmUzYTA1ZGM4Mjk2N2QzOTMzOWZiNTJlZTNiZjJiNWVkOGFkZWIyMmVjNTMzMzhmYjliIn19LCJzaWduYXR1cmUiOnsiY29udGVudCI6Ik1FUUNJQlE5WlNkbk9jOW5xZnQ4M0dETmE3ODlrZzBISFQ0YWxtUitFUDdoU2lVa0FpQk04NFl1MnJFNFY5NHZOY1pqUEJobWFWK2tBbzhjbFZEZmNlcmtyc1UwclE9PSIsInB1YmxpY0tleSI6eyJjb250ZW50IjoiTFMwdExTMUNSVWRKVGlCRFJWSlVTVVpKUTBGVVJTMHRMUzB0Q2sxSlNVaExla05EUW5KTFowRjNTVUpCWjBsVlRrTndTM2R4YTJvNVNFWTNMek5yZW5KdWFVbE9TMWQ2ZWxWTmQwTm5XVWxMYjFwSmVtb3dSVUYzVFhjS1RucEZWazFDVFVkQk1WVkZRMmhOVFdNeWJHNWpNMUoyWTIxVmRWcEhWakpOVWpSM1NFRlpSRlpSVVVSRmVGWjZZVmRrZW1SSE9YbGFVekZ3WW01U2JBcGpiVEZzV2tkc2FHUkhWWGRJYUdOT1RXcFplRTFFUlRWTmFrbDZUVlJGTVZkb1kwNU5hbGw0VFVSRk5VMXFTVEJOVkVVeFYycEJRVTFHYTNkRmQxbElDa3R2V2tsNmFqQkRRVkZaU1V0dldrbDZhakJFUVZGalJGRm5RVVV6WldSemRYZHVTamRxY1hGbGJVeDRORUZhYkdkU1dEVkllR2hqVjFKRk5rbE9Va1lLZFhWSVltMWFTa05zWm5ndmFYQnFiWE0yYmxKQllWaEZXR05TUVZOWE0yeExjMGx3V1dWVGNpOVVNVXRGWWpBd0x6WlBRMEprUlhkbloxaE9UVUUwUndwQk1WVmtSSGRGUWk5M1VVVkJkMGxJWjBSQlZFSm5UbFpJVTFWRlJFUkJTMEpuWjNKQ1owVkdRbEZqUkVGNlFXWkNaMDVXU0ZOTlJVZEVRVmRuUWxSbUNqQXJibEJXYVZGU2JIWnRiekpQYTI5V1lVeEhUR2hvYTFCNlFuSkNaMDVXU0ZKRlFrRm1PRVZaVkVKbWFHd3hiMlJJVW5kamVtOTJUREprY0dSSGFERUtXV2sxYW1JeU1IWmhTRlp1V2pKc2RWb3lXbWhaTWxWMllUSldlV0p0Vm5OamVURnFZakl4ZEdSWE5YQmtTR3QyVEcxa2NHUkhhREZaYVRrellqTktjZ3BhYlhoMlpETk5kbGx1Vm5CaVIxRjFaVmRHZEdKRlFubGFWMXA2VERKb2JGbFhVbnBNTWpGb1lWYzBkMDlSV1V0TGQxbENRa0ZIUkhaNlFVSkJVVkZ5Q21GSVVqQmpTRTAyVEhrNU1HSXlkR3hpYVRWb1dUTlNjR0l5TlhwTWJXUndaRWRvTVZsdVZucGFXRXBxWWpJMU1GcFhOVEJNYlU1MllsUkJaa0puYjNJS1FtZEZSVUZaVHk5TlFVVkRRa0pHTTJJelNuSmFiWGgyWkRFNWEyRllUbmRaV0ZKcVlVUkJNa0puYjNKQ1owVkZRVmxQTDAxQlJVUkNRMmh0V2xSS2FRcFphbGt5V20xV2JFNXRUWGxaVkdNeldsZE9hbHBxUVRSWlZFRXhUVVJvYTFsNlZUTk9SRWswV2tSUk5FOUhWbXROUWsxSFEybHpSMEZSVVVKbk56aDNDa0ZSVVVWQ1ZVb3hZVmQ0YTAxRGMwZERhWE5IUVZGUlFtYzNPSGRCVVZWRlNGZG9NVm95WkhCaWJXUnRXVmRPYkV3eWRHeGpiVFZzWWtoTmRGa3lPWFFLWWxoV2RXRllValZOUWpCSFEybHpSMEZSVVVKbk56aDNRVkZaUlVRelNteGFiazEyWVVkV2FGcElUWFppVjBad1ltcEJOMEpuYjNKQ1owVkZRVmxQTHdwTlFVVkpRa013VFVzeWFEQmtTRUo2VDJrNGRtUkhPWEphVnpSMVdWZE9NR0ZYT1hWamVUVnVZVmhTYjJSWFNqRmpNbFo1V1RJNWRXUkhWblZrUXpWcUNtSXlNSGRpVVZsTFMzZFpRa0pCUjBSMmVrRkNRMUZTWmtSR01XOWtTRkozWTNwdmRrd3laSEJrUjJneFdXazFhbUl5TUhaaFNGWnVXakpzZFZveVdtZ0tXVEpWZG1FeVZubGliVlp6WTNreGFtSXlNWFJrVnpWd1pFaHJka3h0WkhCa1IyZ3hXV2s1TTJJelNuSmFiWGgyWkROTmRsbHVWbkJpUjFGMVpWZEdkQXBpUlVKNVdsZGFla3d5YUd4WlYxSjZUREl4YUdGWE5IZFBRVmxMUzNkWlFrSkJSMFIyZWtGQ1EyZFJjVVJEYUcxYVZFcHBXV3BaTWxwdFZteE9iVTE1Q2xsVVl6TmFWMDVxV21wQk5GbFVRVEZOUkdocldYcFZNMDVFU1RSYVJGRTBUMGRXYTAxQ2MwZERhWE5IUVZGUlFtYzNPSGRCVVhORlJGRjNUR015Vm5NS1dta3hiMkl6VGpCYVYxRjNVVUZaUzB0M1dVSkNRVWRFZG5wQlFrUkJVWGxFUkVKdlpFaFNkMk42YjNaTU1tUndaRWRvTVZscE5XcGlNakIyWVVoV2JncGFNbXgxV2pKYWFGa3lWWFpoTWxaNVltMVdjMk41TVdwaU1qRjBaRmMxY0dSSWEzZFBRVmxMUzNkWlFrSkJSMFIyZWtGQ1JGRlJjVVJEYUcxYVZFcHBDbGxxV1RKYWJWWnNUbTFOZVZsVVl6TmFWMDVxV21wQk5GbFVRVEZOUkdocldYcFZNMDVFU1RSYVJGRTBUMGRXYTAxQ09FZERhWE5IUVZGUlFtYzNPSGNLUVZFMFJVVlJkMUJqYlZadFkzazViMXBYUm10amVUbDBXVmRzZFUxQ2IwZERhWE5IUVZGUlFtYzNPSGRCVVRoRlJFRjNTMDFVUVROTlZGRXpUbFJWZVFwUFZFRjFRbWR2Y2tKblJVVkJXVTh2VFVGRlVVSkRRVTFJYldnd1pFaENlazlwT0haYU1td3dZVWhXYVV4dFRuWmlVemx2WkZka2JtRlhOVzVhYlVacUNscFVRVmxDWjI5eVFtZEZSVUZaVHk5TlFVVlNRa0Z2VFVORVNURk9la2wzVG5wUmVrMUhNRWREYVhOSFFWRlJRbWMzT0hkQlVrbEZXSGQ0WkdGSVVqQUtZMGhOTmt4NU9XNWhXRkp2WkZkSmRWa3lPWFJNTW1neFdqSmtjR0p0WkcxWlYwNXNUREowYkdOdE5XeGlTRTEwV1RJNWRHSllWblZoV0ZJMVRIazFiZ3BoV0ZKdlpGZEpkbVF5T1hsaE1scHpZak5rZWt3eVNqRmhWM2hyVEc1c2FHSlhlRUZqYlZadFkzazViMXBYUm10amVUbDBXVmRzZFUxRVowZERhWE5IQ2tGUlVVSm5OemgzUVZKTlJVdG5kMjlhYlZWNVdXMUpNazV0V214YVZGcHFUVzFGTTA0eVZtcFpNbGwzVDBkRmQwNVVRVFJhUjAweFRucFJlVTlIVVRBS1QwUm9iRnBFUVdoQ1oyOXlRbWRGUlVGWlR5OU5RVVZWUWtKTlRVVllaSFpqYlhSdFlrYzVNMWd5VW5Cak0wSm9aRWRPYjAxSFVVZERhWE5IUVZGUlFncG5OemgzUVZKVlJWWm5lRlZoU0ZJd1kwaE5Oa3g1T1c1aFdGSnZaRmRKZFZreU9YUk1NbWd4V2pKa2NHSnRaRzFaVjA1c1RESjBiR050Tld4aVNFMTBDbGt5T1hSaVdGWjFZVmhTTlV3eVJtcGtSMngyWW01TmRtTnVWblZqZVRoNlQxUkplazlVVlhoUFJFVjNUa001YUdSSVVteGlXRUl3WTNrNGVFMUNXVWNLUTJselIwRlJVVUpuTnpoM1FWSlpSVU5CZDBkalNGWnBZa2RzYWsxRldVZERhWE5IUVZGUlFtYzNPSGRCVW1kRlQwRjNNbU50Vm5kaWVuQnZaRmRrYmdwaFZ6VnVXbTFHYWxwVE9YSmFXRXAxV2xkNGVreFhUblppVnpFeFltMXNNR1ZVY0hsYVYxazJZMjFXYldONU9XOWFWMFpyWTNrNWRGbFhiSFZOU1VkTENrSm5iM0pDWjBWRlFXUmFOVUZuVVVOQ1NIZEZaV2RDTkVGSVdVRXpWREIzWVhOaVNFVlVTbXBIVWpSamJWZGpNMEZ4U2t0WWNtcGxVRXN6TDJnMGNIa0taME00Y0Rkdk5FRkJRVWRvVm10d1lUWkJRVUZDUVUxQlVucENSa0ZwUlVGd1VYZERRMWc1Tm1KcGJXTlRObmhTU2t0WFNrSnpWV2xUYjBSMlpDdFFhd292VVZsWlJtTnJkRk5XYTBOSlJVbzVMekp5UjFvNGEyeFljMng1ZEZaclpVdEdVWHB5SzJVMGJEZE9iMjkwVkRKRWRUaDFWbWhoWTAxQmIwZERRM0ZIQ2xOTk5EbENRVTFFUVRKalFVMUhVVU5OUWtGRlQyMXhaR2t3UkRoU01sWXhjVTFuTkM5UGVYSjJWRXBYZFRSdllXd3dTMFZsWmxkc2RUWkhSamR6YjJVS1VGTklhMEV3TjFwNVJXbHpkR1UwUWtSblNYZG1OV1JYVG5BM2RtbFFaMFJPVTJaMVluVTJTSE42Y2trdk5FWm5ZVTF3TldoUVFVaG1OV1l2TDFWQk5RcHVUV0puUlU1dVdWWjJlRmRHUzBKRFZsbzRhZ290TFMwdExVVk9SQ0JEUlZKVVNVWkpRMEZVUlMwdExTMHRDZz09In19fX0="}], "timestampVerificationData":{"rfc3161Timestamps":[{"signedTimestamp":"MIICyzADAgEAMIICwgYJKoZIhvcNAQcCoIICszCCAq8CAQMxDTALBglghkgBZQMEAgEwgbgGCyqGSIb3DQEJEAEEoIGoBIGlMIGiAgEBBgkrBgEEAYO/MAIwMTANBglghkgBZQMEAgEFAAQgo8yY2H2W5TJzDMi71ksxVvtMI5Yk85L5ePrKyhWWYuMCFQCmdZ6tqY2ljRgQDYyeRbpJ+UjT0hgPMjAyNjEwMTkyMjMxMTZaMAMCAQGgMqQwMC4xFTATBgNVBAoTDHNpZ3N0b3JlLmRldjEVMBMGA1UEAxMMc2lnc3RvcmUtdHNhoAAxggHcMIIB2AIBATBRMDkxFTATBgNVBAoTDHNpZ3N0b3JlLmRldjEgMB4GA1UEAxMXc2lnc3RvcmUtdHNhLXNlbGZzaWduZWQCFDoTVC8MkGHuvMFDL8uKjosqI4sMMAsGCWCGSAFlAwQCAaCB/DAaBgkqhkiG9w0BCQMxDQYLKoZIhvcNAQkQAQQwHAYJKoZIhvcNAQkFMQ8XDTI2MTAxOTIyMzExNlowLwYJKoZIhvcNAQkEMSIEIOsIp7PBFnHyxWzK5qw8TodpDGtaKM4KYOySCt5HM4kaMIGOBgsqhkiG9w0BCRACLzF/MH0wezB5BCCF+Se8B6tiysO0Q1bBDvyBssaIP9p6uebYcNnROs0FtzBVMD2kOzA5MRUwEwYDVQQKEwxzaWdzdG9yZS5kZXYxIDAeBgNVBAMTF3NpZ3N0b3JlLXRzYS1zZWxmc2lnbmVkAhQ6E1QvDJBh7rzBQy/Lio6LKiOLDDAKBggqhkjOPQQDAgRoMGYCMQD799LG2rm5BHbJ+kOEhatzZmFa2atv0WZ5e8hV1h6NdYDUIQYkTYLtLfWUt9FdwEUCMQDd0xGZB1eu/B0kZhmEIO14nniSqN90/Zrh4CHX25WEB+okXRo3Ug41mIWRZeMv0bM="}]}}, "messageSignature":{"messageDigest":{"algorithm":"SHA2_256", "digest":"QIOOKKgi0uOgXcgpZ9OTOftS7jvyte2K3rIuxTM4+5s="}, "signature":"MEQCIBQ9ZSdnOc9nqft83GDNa789kg0HHT4almR+EP7hSiUkAiBM84Yu2rE4V94vNcZjPBhmaV+kAo8clVDfcerkrsU0rQ=="}}
build/torch212-rocm71-x86_64-linux/mha.py ADDED
@@ -0,0 +1,332 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FlashAttention forward entry points backed by aiter's Composable-Kernel FMHA.
2
+
3
+ This is the compiled (HIP / Composable Kernel) counterpart to the Triton
4
+ ``aiter-flash-attn`` package. Only the *forward* pass is provided (dense and
5
+ variable-length), including support for learnable attention sinks via
6
+ ``window_size[2]`` / ``s_aux`` (e.g. gpt-oss). The op set ships exclusively
7
+ the CK ``mha_fwd`` / ``mha_varlen_fwd`` kernels, so the sink path is always
8
+ taken (no fall-through to a non-sink ASM kernel).
9
+ """
10
+
11
+ from typing import Optional, Tuple, Union
12
+
13
+ import torch
14
+ from torch import Tensor
15
+
16
+ from ._ops import ops
17
+
18
+
19
+ def _round_multiple(x: int, m: int) -> int:
20
+ return (x + m - 1) // m * m
21
+
22
+
23
+ def _maybe_pad_headdim(t: Tensor, target: int) -> Tensor:
24
+ og = t.size(-1)
25
+ if og == target:
26
+ return t
27
+ return torch.nn.functional.pad(t, [0, target - og])
28
+
29
+
30
+ def _unpack_window(window_size) -> Tuple[int, int, int]:
31
+ left = int(window_size[0])
32
+ right = int(window_size[1])
33
+ sink = int(window_size[2]) if len(window_size) == 3 else 0
34
+ return left, right, sink
35
+
36
+
37
+ def _prep_sink(s_aux: Optional[Tensor], nheads: int, device) -> Optional[Tensor]:
38
+ if s_aux is None:
39
+ return None
40
+ assert s_aux.device == device, "s_aux must be on the same device as q"
41
+ assert s_aux.shape[0] == nheads, "s_aux must have shape [nheads_q]"
42
+ if s_aux.dtype != torch.float32:
43
+ s_aux = s_aux.to(torch.float32)
44
+ return s_aux
45
+
46
+
47
+ def _effective_sink_size(sink_size: int, s_aux: Optional[Tensor]) -> int:
48
+ # The CK kernel only takes the sink path when sink_size > 0. Callers using
49
+ # the transformers attention interface pass the per-head sink logits as
50
+ # ``s_aux`` but leave ``window_size`` a 2-tuple (sink_size == 0), so enable
51
+ # a single learnable sink automatically when s_aux is given.
52
+ if s_aux is not None and sink_size == 0:
53
+ return 1
54
+ return sink_size
55
+
56
+
57
+ def flash_attn_func(
58
+ q: Tensor,
59
+ k: Tensor,
60
+ v: Tensor,
61
+ dropout_p: float = 0.0,
62
+ softmax_scale: Optional[float] = None,
63
+ causal: bool = False,
64
+ window_size: Tuple[int, ...] = (-1, -1, 0),
65
+ bias: Optional[Tensor] = None,
66
+ alibi_slopes: Optional[Tensor] = None,
67
+ return_lse: bool = False,
68
+ return_attn_probs: bool = False,
69
+ s_aux: Optional[Tensor] = None,
70
+ **kwargs,
71
+ ) -> Union[Tensor, Tuple[Tensor, ...]]:
72
+ """Dense FlashAttention forward.
73
+
74
+ ``q, k, v`` have shape ``(batch, seqlen, nheads, headdim)``. ``window_size``
75
+ is ``(left, right, sink_size)``; ``s_aux`` is an optional ``(nheads_q,)``
76
+ fp32 tensor of learnable sink logits. Extra keyword arguments (e.g. those
77
+ passed by the transformers flash-attention interface) are ignored.
78
+ """
79
+ if softmax_scale is None:
80
+ softmax_scale = q.shape[-1] ** (-0.5)
81
+ window_size_left, window_size_right, sink_size = _unpack_window(window_size)
82
+
83
+ nheads_q = q.size(2)
84
+ s_aux = _prep_sink(s_aux, nheads_q, q.device)
85
+ sink_size = _effective_sink_size(sink_size, s_aux)
86
+
87
+ head_q_og = q.size(-1)
88
+ head_v_og = v.size(-1)
89
+ head_q = _round_multiple(head_q_og, 8)
90
+ head_v = _round_multiple(head_v_og, 8)
91
+ q = _maybe_pad_headdim(q, head_q)
92
+ k = _maybe_pad_headdim(k, head_q)
93
+ v = _maybe_pad_headdim(v, head_v)
94
+
95
+ return_softmax = return_attn_probs and dropout_p > 0
96
+ out, softmax_lse, S_dmask, _ = ops.mha_fwd(
97
+ q,
98
+ k,
99
+ v,
100
+ dropout_p,
101
+ softmax_scale,
102
+ causal,
103
+ window_size_left,
104
+ window_size_right,
105
+ sink_size,
106
+ return_lse or return_attn_probs,
107
+ return_softmax,
108
+ None, # cu_seqlens_q
109
+ None, # cu_seqlens_kv
110
+ None, # out
111
+ bias,
112
+ alibi_slopes,
113
+ None, # q_descale
114
+ None, # k_descale
115
+ None, # v_descale
116
+ s_aux,
117
+ None, # gen
118
+ )
119
+
120
+ out = out[..., :head_v_og]
121
+ if not (return_lse or return_attn_probs):
122
+ return out
123
+ result = (out,)
124
+ if return_lse:
125
+ result = result + (softmax_lse,)
126
+ if return_attn_probs:
127
+ result = result + (S_dmask,)
128
+ return result
129
+
130
+
131
+ def flash_attn_varlen_func(
132
+ q: Tensor,
133
+ k: Tensor,
134
+ v: Tensor,
135
+ cu_seqlens_q: Tensor,
136
+ cu_seqlens_k: Tensor,
137
+ max_seqlen_q: int,
138
+ max_seqlen_k: int,
139
+ dropout_p: float = 0.0,
140
+ softmax_scale: Optional[float] = None,
141
+ causal: bool = False,
142
+ window_size: Tuple[int, ...] = (-1, -1, 0),
143
+ bias: Optional[Tensor] = None,
144
+ alibi_slopes: Optional[Tensor] = None,
145
+ return_lse: bool = False,
146
+ return_attn_probs: bool = False,
147
+ block_table: Optional[Tensor] = None,
148
+ min_seqlen_q: int = 0,
149
+ logits_soft_cap: float = 0.0,
150
+ zero_tensors: bool = False,
151
+ cu_seqlens_q_padded: Optional[Tensor] = None,
152
+ cu_seqlens_k_padded: Optional[Tensor] = None,
153
+ s_aux: Optional[Tensor] = None,
154
+ **kwargs,
155
+ ) -> Union[Tensor, Tuple[Tensor, ...]]:
156
+ """Variable-length / packed FlashAttention forward.
157
+
158
+ ``q`` has shape ``(total_q, nheads, headdim)``; ``cu_seqlens_*`` are int32
159
+ cumulative offsets of shape ``(batch + 1,)``. Sinks are supported through
160
+ ``window_size[2]`` / ``s_aux`` as in :func:`flash_attn_func`. Extra keyword
161
+ arguments (e.g. from the transformers flash-attention interface) are ignored.
162
+ """
163
+ if softmax_scale is None:
164
+ softmax_scale = q.shape[-1] ** (-0.5)
165
+ window_size_left, window_size_right, sink_size = _unpack_window(window_size)
166
+
167
+ nheads_q = q.size(1)
168
+ s_aux = _prep_sink(s_aux, nheads_q, q.device)
169
+ sink_size = _effective_sink_size(sink_size, s_aux)
170
+
171
+ head_q_og = q.size(-1)
172
+ head_v_og = v.size(-1)
173
+ head_q = _round_multiple(head_q_og, 8)
174
+ head_v = _round_multiple(head_v_og, 8)
175
+ q = _maybe_pad_headdim(q, head_q)
176
+ k = _maybe_pad_headdim(k, head_q)
177
+ v = _maybe_pad_headdim(v, head_v)
178
+
179
+ return_softmax = return_attn_probs and dropout_p > 0
180
+ out, softmax_lse, S_dmask, _ = ops.mha_varlen_fwd(
181
+ q,
182
+ k,
183
+ v,
184
+ cu_seqlens_q,
185
+ cu_seqlens_k,
186
+ max_seqlen_q,
187
+ max_seqlen_k,
188
+ min_seqlen_q,
189
+ dropout_p,
190
+ softmax_scale,
191
+ logits_soft_cap,
192
+ zero_tensors,
193
+ causal,
194
+ window_size_left,
195
+ window_size_right,
196
+ sink_size,
197
+ return_lse or return_attn_probs,
198
+ return_softmax,
199
+ None, # out
200
+ block_table,
201
+ bias,
202
+ alibi_slopes,
203
+ None, # q_descale
204
+ None, # k_descale
205
+ None, # v_descale
206
+ None, # gen
207
+ cu_seqlens_q_padded,
208
+ cu_seqlens_k_padded,
209
+ s_aux,
210
+ )
211
+
212
+ out = out[..., :head_v_og]
213
+ if not (return_lse or return_attn_probs):
214
+ return out
215
+ result = (out,)
216
+ if return_lse:
217
+ result = result + (softmax_lse,)
218
+ if return_attn_probs:
219
+ result = result + (S_dmask,)
220
+ return result
221
+
222
+
223
+ def _append_to_contiguous_cache(k_cache, v_cache, k, v, cache_seqlens, Sq):
224
+ """In-place append of new k/v into a contiguous cache at per-batch offsets."""
225
+ B = k_cache.shape[0]
226
+ if isinstance(cache_seqlens, int):
227
+ cache_seqlens = torch.full((B,), cache_seqlens, device=k_cache.device, dtype=torch.long)
228
+ pos = cache_seqlens.view(B, 1).to(torch.long) + torch.arange(Sq, device=k_cache.device).view(1, Sq)
229
+ bidx = torch.arange(B, device=k_cache.device).view(B, 1).expand(B, Sq)
230
+ k_cache[bidx, pos] = k.to(k_cache.dtype)
231
+ v_cache[bidx, pos] = v.to(v_cache.dtype)
232
+
233
+
234
+ def flash_attn_with_kvcache(
235
+ q: Tensor,
236
+ k_cache: Tensor,
237
+ v_cache: Tensor,
238
+ k: Optional[Tensor] = None,
239
+ v: Optional[Tensor] = None,
240
+ rotary_cos: Optional[Tensor] = None,
241
+ rotary_sin: Optional[Tensor] = None,
242
+ cache_seqlens=None,
243
+ block_table: Optional[Tensor] = None,
244
+ softmax_scale: Optional[float] = None,
245
+ causal: bool = True,
246
+ window_size: Tuple[int, ...] = (-1, -1, 0),
247
+ s_aux: Optional[Tensor] = None,
248
+ return_lse: bool = False,
249
+ **kwargs,
250
+ ) -> Union[Tensor, Tuple[Tensor, ...]]:
251
+ """FlashAttention against a KV cache, built on the CK split-KV kernel.
252
+
253
+ ``q`` is ``(batch, seqlen_q, nheads, headdim)``. For a contiguous cache,
254
+ ``k_cache``/``v_cache`` are ``(batch, seqlen_cache, nheads_k, headdim)`` and
255
+ ``cache_seqlens`` (int or ``(batch,)``) is the number of valid cache entries
256
+ *before* appending. If ``k``/``v`` (the new tokens) are given, they are
257
+ written into the cache in place. The query attends to the valid prefix
258
+ (``cache_seqlens`` + ``seqlen_q``) with bottom-right causal alignment.
259
+
260
+ Notes / current limitations:
261
+ - Rotary embedding is not applied here; pass ``q``/``k`` already rotated
262
+ (``rotary_cos``/``rotary_sin`` are rejected).
263
+ - Paged caches are addressed via ``block_table`` (forwarded to the split-KV
264
+ varlen kernel); in-cache append for paged layouts must be done by the
265
+ caller (pass ``k=v=None``).
266
+ """
267
+ if rotary_cos is not None or rotary_sin is not None:
268
+ raise NotImplementedError(
269
+ "flash_attn_with_kvcache does not apply rotary; rotate q/k beforehand."
270
+ )
271
+ if softmax_scale is None:
272
+ softmax_scale = q.shape[-1] ** (-0.5)
273
+ window_size_left, window_size_right, sink_size = _unpack_window(window_size)
274
+ B, Sq, Hq, D = q.shape
275
+ s_aux = _prep_sink(s_aux, Hq, q.device)
276
+ sink_size = _effective_sink_size(sink_size, s_aux)
277
+
278
+ paged = block_table is not None
279
+
280
+ # ---- append new tokens into the cache (contiguous layout only) ----
281
+ if k is not None and not paged:
282
+ assert cache_seqlens is not None, "cache_seqlens required to append into the cache"
283
+ _append_to_contiguous_cache(k_cache, v_cache, k, v, cache_seqlens, Sq)
284
+ elif k is not None and paged:
285
+ raise NotImplementedError(
286
+ "in-cache append for paged layouts is not supported; append externally "
287
+ "and call with k=v=None."
288
+ )
289
+
290
+ appended = 0 if k is None else Sq
291
+
292
+ # ---- key lengths ----
293
+ if cache_seqlens is None:
294
+ klens = torch.full((B,), k_cache.shape[1], device=q.device, dtype=torch.long)
295
+ elif isinstance(cache_seqlens, int):
296
+ klens = torch.full((B,), cache_seqlens + appended, device=q.device, dtype=torch.long)
297
+ else:
298
+ klens = cache_seqlens.to(q.device, torch.long) + appended
299
+
300
+ uniform = bool((klens == klens[0]).all().item())
301
+
302
+ # ---- fast path: contiguous cache, uniform length -> dense kernel ----
303
+ if not paged and uniform:
304
+ L = int(klens[0].item())
305
+ return flash_attn_func(
306
+ q, k_cache[:, :L], v_cache[:, :L],
307
+ softmax_scale=softmax_scale, causal=causal,
308
+ window_size=(window_size_left, window_size_right, sink_size),
309
+ s_aux=s_aux, return_lse=return_lse,
310
+ )
311
+
312
+ # ---- general path: variable cache lengths -> split-KV varlen kernel ----
313
+ cu_q = torch.arange(0, (B + 1) * Sq, Sq, device=q.device, dtype=torch.int32)
314
+ cu_k = torch.zeros(B + 1, device=q.device, dtype=torch.int32)
315
+ cu_k[1:] = torch.cumsum(klens, 0).to(torch.int32)
316
+ q_packed = q.reshape(B * Sq, Hq, D)
317
+ if paged:
318
+ k_packed, v_packed = k_cache, v_cache # kernel gathers via block_table
319
+ else:
320
+ k_packed = torch.cat([k_cache[b, : int(klens[b])] for b in range(B)], dim=0)
321
+ v_packed = torch.cat([v_cache[b, : int(klens[b])] for b in range(B)], dim=0)
322
+
323
+ out = flash_attn_varlen_func(
324
+ q_packed, k_packed, v_packed, cu_q, cu_k, Sq, int(klens.max().item()),
325
+ softmax_scale=softmax_scale, causal=causal,
326
+ window_size=(window_size_left, window_size_right, sink_size),
327
+ block_table=block_table, s_aux=s_aux, return_lse=return_lse,
328
+ )
329
+ if return_lse:
330
+ out, lse = out
331
+ return out.reshape(B, Sq, Hq, D), lse
332
+ return out.reshape(B, Sq, Hq, D)
build/torch212-rocm71-x86_64-linux/symbols.json ADDED
@@ -0,0 +1,349 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "schema_version": 1,
3
+ "module": "_aiter_flash_attn_ck_rocm_fe2bb66",
4
+ "functions": [
5
+ {
6
+ "name": "flash_attn_func",
7
+ "qualname": "flash_attn_func",
8
+ "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha",
9
+ "kind": "function",
10
+ "doc": "Dense FlashAttention forward.\n\n``q, k, v`` have shape ``(batch, seqlen, nheads, headdim)``. ``window_size``\nis ``(left, right, sink_size)``; ``s_aux`` is an optional ``(nheads_q,)``\nfp32 tensor of learnable sink logits. Extra keyword arguments (e.g. those\npassed by the transformers flash-attention interface) are ignored.",
11
+ "signature": {
12
+ "parameters": [
13
+ {
14
+ "name": "q",
15
+ "kind": "POSITIONAL_OR_KEYWORD",
16
+ "default": null,
17
+ "annotation": "torch.Tensor"
18
+ },
19
+ {
20
+ "name": "k",
21
+ "kind": "POSITIONAL_OR_KEYWORD",
22
+ "default": null,
23
+ "annotation": "torch.Tensor"
24
+ },
25
+ {
26
+ "name": "v",
27
+ "kind": "POSITIONAL_OR_KEYWORD",
28
+ "default": null,
29
+ "annotation": "torch.Tensor"
30
+ },
31
+ {
32
+ "name": "dropout_p",
33
+ "kind": "POSITIONAL_OR_KEYWORD",
34
+ "default": "0.0",
35
+ "annotation": "float"
36
+ },
37
+ {
38
+ "name": "softmax_scale",
39
+ "kind": "POSITIONAL_OR_KEYWORD",
40
+ "default": "None",
41
+ "annotation": "float | None"
42
+ },
43
+ {
44
+ "name": "causal",
45
+ "kind": "POSITIONAL_OR_KEYWORD",
46
+ "default": "False",
47
+ "annotation": "bool"
48
+ },
49
+ {
50
+ "name": "window_size",
51
+ "kind": "POSITIONAL_OR_KEYWORD",
52
+ "default": "(-1, -1, 0)",
53
+ "annotation": "Tuple[int, ...]"
54
+ },
55
+ {
56
+ "name": "bias",
57
+ "kind": "POSITIONAL_OR_KEYWORD",
58
+ "default": "None",
59
+ "annotation": "torch.Tensor | None"
60
+ },
61
+ {
62
+ "name": "alibi_slopes",
63
+ "kind": "POSITIONAL_OR_KEYWORD",
64
+ "default": "None",
65
+ "annotation": "torch.Tensor | None"
66
+ },
67
+ {
68
+ "name": "return_lse",
69
+ "kind": "POSITIONAL_OR_KEYWORD",
70
+ "default": "False",
71
+ "annotation": "bool"
72
+ },
73
+ {
74
+ "name": "return_attn_probs",
75
+ "kind": "POSITIONAL_OR_KEYWORD",
76
+ "default": "False",
77
+ "annotation": "bool"
78
+ },
79
+ {
80
+ "name": "s_aux",
81
+ "kind": "POSITIONAL_OR_KEYWORD",
82
+ "default": "None",
83
+ "annotation": "torch.Tensor | None"
84
+ },
85
+ {
86
+ "name": "kwargs",
87
+ "kind": "VAR_KEYWORD",
88
+ "default": null,
89
+ "annotation": null
90
+ }
91
+ ],
92
+ "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]"
93
+ }
94
+ },
95
+ {
96
+ "name": "flash_attn_varlen_func",
97
+ "qualname": "flash_attn_varlen_func",
98
+ "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha",
99
+ "kind": "function",
100
+ "doc": "Variable-length / packed FlashAttention forward.\n\n``q`` has shape ``(total_q, nheads, headdim)``; ``cu_seqlens_*`` are int32\ncumulative offsets of shape ``(batch + 1,)``. Sinks are supported through\n``window_size[2]`` / ``s_aux`` as in :func:`flash_attn_func`. Extra keyword\narguments (e.g. from the transformers flash-attention interface) are ignored.",
101
+ "signature": {
102
+ "parameters": [
103
+ {
104
+ "name": "q",
105
+ "kind": "POSITIONAL_OR_KEYWORD",
106
+ "default": null,
107
+ "annotation": "torch.Tensor"
108
+ },
109
+ {
110
+ "name": "k",
111
+ "kind": "POSITIONAL_OR_KEYWORD",
112
+ "default": null,
113
+ "annotation": "torch.Tensor"
114
+ },
115
+ {
116
+ "name": "v",
117
+ "kind": "POSITIONAL_OR_KEYWORD",
118
+ "default": null,
119
+ "annotation": "torch.Tensor"
120
+ },
121
+ {
122
+ "name": "cu_seqlens_q",
123
+ "kind": "POSITIONAL_OR_KEYWORD",
124
+ "default": null,
125
+ "annotation": "torch.Tensor"
126
+ },
127
+ {
128
+ "name": "cu_seqlens_k",
129
+ "kind": "POSITIONAL_OR_KEYWORD",
130
+ "default": null,
131
+ "annotation": "torch.Tensor"
132
+ },
133
+ {
134
+ "name": "max_seqlen_q",
135
+ "kind": "POSITIONAL_OR_KEYWORD",
136
+ "default": null,
137
+ "annotation": "int"
138
+ },
139
+ {
140
+ "name": "max_seqlen_k",
141
+ "kind": "POSITIONAL_OR_KEYWORD",
142
+ "default": null,
143
+ "annotation": "int"
144
+ },
145
+ {
146
+ "name": "dropout_p",
147
+ "kind": "POSITIONAL_OR_KEYWORD",
148
+ "default": "0.0",
149
+ "annotation": "float"
150
+ },
151
+ {
152
+ "name": "softmax_scale",
153
+ "kind": "POSITIONAL_OR_KEYWORD",
154
+ "default": "None",
155
+ "annotation": "float | None"
156
+ },
157
+ {
158
+ "name": "causal",
159
+ "kind": "POSITIONAL_OR_KEYWORD",
160
+ "default": "False",
161
+ "annotation": "bool"
162
+ },
163
+ {
164
+ "name": "window_size",
165
+ "kind": "POSITIONAL_OR_KEYWORD",
166
+ "default": "(-1, -1, 0)",
167
+ "annotation": "Tuple[int, ...]"
168
+ },
169
+ {
170
+ "name": "bias",
171
+ "kind": "POSITIONAL_OR_KEYWORD",
172
+ "default": "None",
173
+ "annotation": "torch.Tensor | None"
174
+ },
175
+ {
176
+ "name": "alibi_slopes",
177
+ "kind": "POSITIONAL_OR_KEYWORD",
178
+ "default": "None",
179
+ "annotation": "torch.Tensor | None"
180
+ },
181
+ {
182
+ "name": "return_lse",
183
+ "kind": "POSITIONAL_OR_KEYWORD",
184
+ "default": "False",
185
+ "annotation": "bool"
186
+ },
187
+ {
188
+ "name": "return_attn_probs",
189
+ "kind": "POSITIONAL_OR_KEYWORD",
190
+ "default": "False",
191
+ "annotation": "bool"
192
+ },
193
+ {
194
+ "name": "block_table",
195
+ "kind": "POSITIONAL_OR_KEYWORD",
196
+ "default": "None",
197
+ "annotation": "torch.Tensor | None"
198
+ },
199
+ {
200
+ "name": "min_seqlen_q",
201
+ "kind": "POSITIONAL_OR_KEYWORD",
202
+ "default": "0",
203
+ "annotation": "int"
204
+ },
205
+ {
206
+ "name": "logits_soft_cap",
207
+ "kind": "POSITIONAL_OR_KEYWORD",
208
+ "default": "0.0",
209
+ "annotation": "float"
210
+ },
211
+ {
212
+ "name": "zero_tensors",
213
+ "kind": "POSITIONAL_OR_KEYWORD",
214
+ "default": "False",
215
+ "annotation": "bool"
216
+ },
217
+ {
218
+ "name": "cu_seqlens_q_padded",
219
+ "kind": "POSITIONAL_OR_KEYWORD",
220
+ "default": "None",
221
+ "annotation": "torch.Tensor | None"
222
+ },
223
+ {
224
+ "name": "cu_seqlens_k_padded",
225
+ "kind": "POSITIONAL_OR_KEYWORD",
226
+ "default": "None",
227
+ "annotation": "torch.Tensor | None"
228
+ },
229
+ {
230
+ "name": "s_aux",
231
+ "kind": "POSITIONAL_OR_KEYWORD",
232
+ "default": "None",
233
+ "annotation": "torch.Tensor | None"
234
+ },
235
+ {
236
+ "name": "kwargs",
237
+ "kind": "VAR_KEYWORD",
238
+ "default": null,
239
+ "annotation": null
240
+ }
241
+ ],
242
+ "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]"
243
+ }
244
+ },
245
+ {
246
+ "name": "flash_attn_with_kvcache",
247
+ "qualname": "flash_attn_with_kvcache",
248
+ "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha",
249
+ "kind": "function",
250
+ "doc": "FlashAttention against a KV cache, built on the CK split-KV kernel.\n\n``q`` is ``(batch, seqlen_q, nheads, headdim)``. For a contiguous cache,\n``k_cache``/``v_cache`` are ``(batch, seqlen_cache, nheads_k, headdim)`` and\n``cache_seqlens`` (int or ``(batch,)``) is the number of valid cache entries\n*before* appending. If ``k``/``v`` (the new tokens) are given, they are\nwritten into the cache in place. The query attends to the valid prefix\n(``cache_seqlens`` + ``seqlen_q``) with bottom-right causal alignment.\n\nNotes / current limitations:\n- Rotary embedding is not applied here; pass ``q``/``k`` already rotated\n (``rotary_cos``/``rotary_sin`` are rejected).\n- Paged caches are addressed via ``block_table`` (forwarded to the split-KV\n varlen kernel); in-cache append for paged layouts must be done by the\n caller (pass ``k=v=None``).",
251
+ "signature": {
252
+ "parameters": [
253
+ {
254
+ "name": "q",
255
+ "kind": "POSITIONAL_OR_KEYWORD",
256
+ "default": null,
257
+ "annotation": "torch.Tensor"
258
+ },
259
+ {
260
+ "name": "k_cache",
261
+ "kind": "POSITIONAL_OR_KEYWORD",
262
+ "default": null,
263
+ "annotation": "torch.Tensor"
264
+ },
265
+ {
266
+ "name": "v_cache",
267
+ "kind": "POSITIONAL_OR_KEYWORD",
268
+ "default": null,
269
+ "annotation": "torch.Tensor"
270
+ },
271
+ {
272
+ "name": "k",
273
+ "kind": "POSITIONAL_OR_KEYWORD",
274
+ "default": "None",
275
+ "annotation": "torch.Tensor | None"
276
+ },
277
+ {
278
+ "name": "v",
279
+ "kind": "POSITIONAL_OR_KEYWORD",
280
+ "default": "None",
281
+ "annotation": "torch.Tensor | None"
282
+ },
283
+ {
284
+ "name": "rotary_cos",
285
+ "kind": "POSITIONAL_OR_KEYWORD",
286
+ "default": "None",
287
+ "annotation": "torch.Tensor | None"
288
+ },
289
+ {
290
+ "name": "rotary_sin",
291
+ "kind": "POSITIONAL_OR_KEYWORD",
292
+ "default": "None",
293
+ "annotation": "torch.Tensor | None"
294
+ },
295
+ {
296
+ "name": "cache_seqlens",
297
+ "kind": "POSITIONAL_OR_KEYWORD",
298
+ "default": "None",
299
+ "annotation": null
300
+ },
301
+ {
302
+ "name": "block_table",
303
+ "kind": "POSITIONAL_OR_KEYWORD",
304
+ "default": "None",
305
+ "annotation": "torch.Tensor | None"
306
+ },
307
+ {
308
+ "name": "softmax_scale",
309
+ "kind": "POSITIONAL_OR_KEYWORD",
310
+ "default": "None",
311
+ "annotation": "float | None"
312
+ },
313
+ {
314
+ "name": "causal",
315
+ "kind": "POSITIONAL_OR_KEYWORD",
316
+ "default": "True",
317
+ "annotation": "bool"
318
+ },
319
+ {
320
+ "name": "window_size",
321
+ "kind": "POSITIONAL_OR_KEYWORD",
322
+ "default": "(-1, -1, 0)",
323
+ "annotation": "Tuple[int, ...]"
324
+ },
325
+ {
326
+ "name": "s_aux",
327
+ "kind": "POSITIONAL_OR_KEYWORD",
328
+ "default": "None",
329
+ "annotation": "torch.Tensor | None"
330
+ },
331
+ {
332
+ "name": "return_lse",
333
+ "kind": "POSITIONAL_OR_KEYWORD",
334
+ "default": "False",
335
+ "annotation": "bool"
336
+ },
337
+ {
338
+ "name": "kwargs",
339
+ "kind": "VAR_KEYWORD",
340
+ "default": null,
341
+ "annotation": null
342
+ }
343
+ ],
344
+ "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]"
345
+ }
346
+ }
347
+ ],
348
+ "layers": []
349
+ }
build/torch212-rocm72-x86_64-linux/__init__.py ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """AITER Composable-Kernel FlashAttention kernels for AMD ROCm (compiled HIP).
2
+
3
+ Compiled (Composable Kernel / HIP) counterpart to the Triton-based
4
+ ``aiter-flash-attn`` package, repackaged from the CK FMHA forward path of the
5
+ `ROCm/aiter <https://github.com/ROCm/aiter>`_ project as a self-contained
6
+ Hugging Face Hub kernel. Exposes the FlashAttention forward entry points with
7
+ learnable attention sink support (e.g. gpt-oss).
8
+ """
9
+
10
+ from .mha import (
11
+ flash_attn_func,
12
+ flash_attn_varlen_func,
13
+ flash_attn_with_kvcache,
14
+ )
15
+
16
+
17
+ __kernel_metadata__ = {
18
+ "license": "mit",
19
+ }
20
+
21
+
22
+ __all__ = [
23
+ "__kernel_metadata__",
24
+ "flash_attn_func",
25
+ "flash_attn_varlen_func",
26
+ "flash_attn_with_kvcache",
27
+ ]
build/torch212-rocm72-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7b2047a691f2945147456fef163603452d23707ec638cf2ea76fde4f596571ba
3
+ size 189896064
build/torch212-rocm72-x86_64-linux/_ops.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _aiter_flash_attn_ck_rocm_fe2bb66
3
+ ops = torch.ops._aiter_flash_attn_ck_rocm_fe2bb66
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ """
7
+ Prefix op by namespace.
8
+ """
9
+ return f"_aiter_flash_attn_ck_rocm_fe2bb66::{op_name}"
build/torch212-rocm72-x86_64-linux/metadata.json ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "aiter-flash-attn-ck",
3
+ "id": "_aiter_flash_attn_ck_rocm_fe2bb66",
4
+ "version": 1,
5
+ "kernels-minver": "0.14.0",
6
+ "license": "MIT",
7
+ "python-depends": [],
8
+ "kernel-depends": [],
9
+ "backend": {
10
+ "type": "rocm",
11
+ "archs": [
12
+ "gfx942",
13
+ "gfx950"
14
+ ]
15
+ },
16
+ "digest": {
17
+ "algorithm": "sha256",
18
+ "files": {
19
+ "__init__.py": "7wj1GTm2bqTLLCyT4xLuAk/+JWYrhINjY/ODMNWhgZ0=",
20
+ "_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so": "eyBHppHylFFHRW/vFjYDRS0jcH7GOM8up2/eT1llcbo=",
21
+ "_ops.py": "4WkdYDWWJp60zep+oW/5ym9vIeVMNfrgRCkP3o/w7yU=",
22
+ "mha.py": "zCAjY4O+DiCzmEL+W9sqXxZaqjsj4RJbWFcyPpUZejk=",
23
+ "symbols.json": "rhnF4H8Jsb6I+YPxIVFMgnLcTQJODEXbZ7ge77NZXAU="
24
+ }
25
+ },
26
+ "provenance": {
27
+ "kernel-builder": {
28
+ "version": "0.18.0-dev0",
29
+ "commit": "d054906a13710b811b96697b4635f03020842733",
30
+ "dirty": false
31
+ },
32
+ "kernel": {
33
+ "commit": "fe2bb66fee6c2a77eccf08a0508dc57428d488ed",
34
+ "dirty": false
35
+ }
36
+ }
37
+ }
build/torch212-rocm72-x86_64-linux/metadata.json.sigstore ADDED
@@ -0,0 +1 @@
 
 
1
+ {"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json", "verificationMaterial":{"certificate":{"rawBytes":"MIIHLTCCBrKgAwIBAgIUQ45I/pFC4gf04I3vc+3x0g7A174wCgYIKoZIzj0EAwMwNzEVMBMGA1UEChMMc2lnc3RvcmUuZGV2MR4wHAYDVQQDExVzaWdzdG9yZS1pbnRlcm1lZGlhdGUwHhcNMjYxMDE5MjIzMTE3WhcNMjYxMDE5MjI0MTE3WjAAMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEzjsfKDVfgfG/aQDPwqQuhwftoZEZK9S5hBiZEa1B80IEyXRgtQ/mvsOwoTYJ9mvxC1+nNSGJdimuEs+feeY6f6OCBdEwggXNMA4GA1UdDwEB/wQEAwIHgDATBgNVHSUEDDAKBggrBgEFBQcDAzAfBgNVHSMEGDAWgBTf0+nPViQRlvmo2OkoVaLGLhhkPzBrBgNVHREBAf8EYTBfhl1odHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkvLmdpdGh1Yi93b3JrZmxvd3MvYnVpbGQueWFtbEByZWZzL2hlYWRzL21haW4wOQYKKwYBBAGDvzABAQQraHR0cHM6Ly90b2tlbi5hY3Rpb25zLmdpdGh1YnVzZXJjb250ZW50LmNvbTAfBgorBgEEAYO/MAECBBF3b3JrZmxvd19kaXNwYXRjaDA2BgorBgEEAYO/MAEDBChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMBMGCisGAQQBg78wAQQEBUJ1aWxkMCsGCisGAQQBg78wAQUEHWh1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5MB0GCisGAQQBg78wAQYED3JlZnMvaGVhZHMvbWFpbjA7BgorBgEEAYO/MAEIBC0MK2h0dHBzOi8vdG9rZW4uYWN0aW9ucy5naXRodWJ1c2VyY29udGVudC5jb20wbQYKKwYBBAGDvzABCQRfDF1odHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkvLmdpdGh1Yi93b3JrZmxvd3MvYnVpbGQueWFtbEByZWZzL2hlYWRzL21haW4wOAYKKwYBBAGDvzABCgQqDChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMBsGCisGAQQBg78wAQsEDQwLc2VsZi1ob3N0ZWQwQAYKKwYBBAGDvzABDAQyDDBodHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkwOAYKKwYBBAGDvzABDQQqDChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMB8GCisGAQQBg78wAQ4EEQwPcmVmcy9oZWFkcy9tYWluMBoGCisGAQQBg78wAQ8EDAwKMTA3MTQ3NTUyOTAuBgorBgEEAYO/MAEQBCAMHmh0dHBzOi8vZ2l0aHViLmNvbS9odWdnaW5nZmFjZTAYBgorBgEEAYO/MAERBAoMCDI1NzIwNzQzMG0GCisGAQQBg78wARIEXwxdaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5Ly5naXRodWIvd29ya2Zsb3dzL2J1aWxkLnlhbWxAcmVmcy9oZWFkcy9tYWluMDgGCisGAQQBg78wARMEKgwoZmUyYmI2NmZlZTZjMmE3N2VjY2YwOGEwNTA4ZGM1NzQyOGQ0ODhlZDAhBgorBgEEAYO/MAEUBBMMEXdvcmtmbG93X2Rpc3BhdGNoMGQGCisGAQQBg78wARUEVgxUaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5L2FjdGlvbnMvcnVucy8zOTIzOTUxODEwNC9hdHRlbXB0cy8xMBYGCisGAQQBg78wARYECAwGcHVibGljMEYGCisGAQQBg78wARgEOAw2cmVwbzpodWdnaW5nZmFjZS9rZXJuZWxzLWNvbW11bml0eTpyZWY6cmVmcy9oZWFkcy9tYWluMIGKBgorBgEEAdZ5AgQCBHwEegB4AHYA3T0wasbHETJjGR4cmWc3AqJKXrjePK3/h4pygC8p7o4AAAGhVkphtgAABAMARzBFAiEA/V/XQ1zoAiy9Jx6LwsG2v9te8QMMdKgo5lxz6/CHUYYCIEBZ2id17T5aK+gexeVyI/uX1eFwmxl5wWBqdSKkHIfWMAoGCCqGSM49BAMDA2kAMGYCMQCtCXUfftdOhO4fxL+pXAyRkt9yANatw2Of1I2oXJa2N5+Ig+NVi5LLNvVkIwur1dMCMQDbzUgi38ARNQqw4NGCku4uStNr02qaF3jeDjSWPjcHfJfFxwDebjKCISt/W2u42Qo="}, "tlogEntries":[{"logIndex":"3327212104", "logId":{"keyId":"wNI9atQGlz+VWfO6LRygH4QUfY/8W4RFwiT5i5WRgB0="}, "kindVersion":{"kind":"hashedrekord", "version":"0.0.1"}, "integratedTime":"1792449077", "inclusionPromise":{"signedEntryTimestamp":"MEYCIQDRlFftrJStusZSVz3NRwN5GmWooDwyhs6XhSTx9F5qyAIhAImpjdodpPfG6f+FY+9Zu/ShY0j1Tdx6nmld19W2EEbc"}, "inclusionProof":{"logIndex":"3205307842", "rootHash":"Zzz8n5Mp9ft5puTfBQIBlCPo6wcO7myrHF6c/DCLCto=", "treeSize":"3205307848", "hashes":["PNwoDRxfezGRvsQGmM0Kf8ZjXCh17m6718g830ihWTY=", "4W/Uty2EfVeMffxMat+M956so0Ydun9CqlURs6q1cTk=", "ZecfiCPMv2u2lSRP0ST3fdJpMAgVpdtdk/HMrR51Tzg=", "RCHihIv1d3ARj25cx7a6BlBmIRIvbqbcYga+rFtfimU=", "NFzeBDxwmGphfWEGD4k9bLG9D0Y0y1ZYfIWGhkLVpVs=", "F4ofUVGe4nSzmh+WwnM4OaVmZh/WQERaH+u/wt8i8FM=", "0aSE/O6jCq82vSwoybvpblRnnP72+1gEInKy6Wyc3E0=", "5egUlZ4Pi0Ix/qLyCDXocsWywJCHqrde/S06vOZvlOM=", "QsXufUpoKtSjKAB19K1G00umQpWvPHlBNuRGFiiVjQQ=", "QIMkwQbwAewhLJd//5FFwI3SnyNQBGjCgJrelfG8vhc=", "Et17CACUv9QN64irUJuBjsrgRGkGnnJ9xacRV/aBdwQ=", "lqgEwkxhDMs/gdIAE4lmjfMwg7nDmv7lda6O0LGliCw=", "HovTPBWzoErvUg1u5sIi3x1ZonZ7yJekVsBrmpomCIA=", "G+nk7mgE5h6dUmv2UY7NDJrY3WvrZY31raaO1AETt+o=", "qlTWNDzERnTBQVcHHXXVPjiUP0SsikGxfcxVlgqVFDo=", "gg9GNApNke3+zNhl4tsWtmIycCFrOMCB3lys6XuSQNQ=", "NlOclldmKPA1tMKIskqHAwuQDYHp31YwpK+OOMrMccc=", "qxzHanAzz57SDdmJe0B7bJK72NTIBbwEMGBBKvDOROw=", "xH/DCseLHr9eKoYT8qsORZK7zVdEGYWHuVtsVrD95wY="], "checkpoint":{"envelope":"rekor.sigstore.dev - 1193050959916656506\n3205307848\nZzz8n5Mp9ft5puTfBQIBlCPo6wcO7myrHF6c/DCLCto=\n\n— rekor.sigstore.dev wNI9ajBEAiAp++VhzyYu83HrKvp+GRExuyer5bXN7+E3gZBsz5SdWwIgf1ZNOW/vRsfmirbx4AdVrRpcwLjtog7l6WYnLs3jdHI=\n"}}, "canonicalizedBody":"eyJhcGlWZXJzaW9uIjoiMC4wLjEiLCJraW5kIjoiaGFzaGVkcmVrb3JkIiwic3BlYyI6eyJkYXRhIjp7Imhhc2giOnsiYWxnb3JpdGhtIjoic2hhMjU2IiwidmFsdWUiOiIzNTcxOTkxYjg4ZDIyMGFkMmUwMjU2OWUyNmIxYjQzY2U5MjAxNjU0M2M1OGE3MWFjOWJiYjEzOTQxZDEwYzQzIn19LCJzaWduYXR1cmUiOnsiY29udGVudCI6Ik1FVUNJRE1QTHZ0UEp2QVZvQ0lrL3dkY1hWSW1NYUNCbW1aNHZ3c2FtN1hiZ0hhcUFpRUFwbTFwOTNZeWdSVHBOdExHeUhYZnFtSnE1MmdnbElYcURKeHY0TEhxMzdVPSIsInB1YmxpY0tleSI6eyJjb250ZW50IjoiTFMwdExTMUNSVWRKVGlCRFJWSlVTVVpKUTBGVVJTMHRMUzB0Q2sxSlNVaE1WRU5EUW5KTFowRjNTVUpCWjBsVlVUUTFTUzl3UmtNMFoyWXdORWt6ZG1Nck0zZ3daemRCTVRjMGQwTm5XVWxMYjFwSmVtb3dSVUYzVFhjS1RucEZWazFDVFVkQk1WVkZRMmhOVFdNeWJHNWpNMUoyWTIxVmRWcEhWakpOVWpSM1NFRlpSRlpSVVVSRmVGWjZZVmRrZW1SSE9YbGFVekZ3WW01U2JBcGpiVEZzV2tkc2FHUkhWWGRJYUdOT1RXcFplRTFFUlRWTmFrbDZUVlJGTTFkb1kwNU5hbGw0VFVSRk5VMXFTVEJOVkVVelYycEJRVTFHYTNkRmQxbElDa3R2V2tsNmFqQkRRVkZaU1V0dldrbDZhakJFUVZGalJGRm5RVVY2YW5ObVMwUldabWRtUnk5aFVVUlFkM0ZSZFdoM1puUnZXa1ZhU3psVE5XaENhVm9LUldFeFFqZ3dTVVY1V0ZKbmRGRXZiWFp6VDNkdlZGbEtPVzEyZUVNeEsyNU9VMGRLWkdsdGRVVnpLMlpsWlZrMlpqWlBRMEprUlhkbloxaE9UVUUwUndwQk1WVmtSSGRGUWk5M1VVVkJkMGxJWjBSQlZFSm5UbFpJVTFWRlJFUkJTMEpuWjNKQ1owVkdRbEZqUkVGNlFXWkNaMDVXU0ZOTlJVZEVRVmRuUWxSbUNqQXJibEJXYVZGU2JIWnRiekpQYTI5V1lVeEhUR2hvYTFCNlFuSkNaMDVXU0ZKRlFrRm1PRVZaVkVKbWFHd3hiMlJJVW5kamVtOTJUREprY0dSSGFERUtXV2sxYW1JeU1IWmhTRlp1V2pKc2RWb3lXbWhaTWxWMllUSldlV0p0Vm5OamVURnFZakl4ZEdSWE5YQmtTR3QyVEcxa2NHUkhhREZaYVRrellqTktjZ3BhYlhoMlpETk5kbGx1Vm5CaVIxRjFaVmRHZEdKRlFubGFWMXA2VERKb2JGbFhVbnBNTWpGb1lWYzBkMDlSV1V0TGQxbENRa0ZIUkhaNlFVSkJVVkZ5Q21GSVVqQmpTRTAyVEhrNU1HSXlkR3hpYVRWb1dUTlNjR0l5TlhwTWJXUndaRWRvTVZsdVZucGFXRXBxWWpJMU1GcFhOVEJNYlU1MllsUkJaa0puYjNJS1FtZEZSVUZaVHk5TlFVVkRRa0pHTTJJelNuSmFiWGgyWkRFNWEyRllUbmRaV0ZKcVlVUkJNa0puYjNKQ1owVkZRVmxQTDAxQlJVUkNRMmh0V2xSS2FRcFphbGt5V20xV2JFNXRUWGxaVkdNeldsZE9hbHBxUVRSWlZFRXhUVVJvYTFsNlZUTk9SRWswV2tSUk5FOUhWbXROUWsxSFEybHpSMEZSVVVKbk56aDNDa0ZSVVVWQ1ZVb3hZVmQ0YTAxRGMwZERhWE5IUVZGUlFtYzNPSGRCVVZWRlNGZG9NVm95WkhCaWJXUnRXVmRPYkV3eWRHeGpiVFZzWWtoTmRGa3lPWFFLWWxoV2RXRllValZOUWpCSFEybHpSMEZSVVVKbk56aDNRVkZaUlVRelNteGFiazEyWVVkV2FGcElUWFppVjBad1ltcEJOMEpuYjNKQ1owVkZRVmxQTHdwTlFVVkpRa013VFVzeWFEQmtTRUo2VDJrNGRtUkhPWEphVnpSMVdWZE9NR0ZYT1hWamVUVnVZVmhTYjJSWFNqRmpNbFo1V1RJNWRXUkhWblZrUXpWcUNtSXlNSGRpVVZsTFMzZFpRa0pCUjBSMmVrRkNRMUZTWmtSR01XOWtTRkozWTNwdmRrd3laSEJrUjJneFdXazFhbUl5TUhaaFNGWnVXakpzZFZveVdtZ0tXVEpWZG1FeVZubGliVlp6WTNreGFtSXlNWFJrVnpWd1pFaHJka3h0WkhCa1IyZ3hXV2s1TTJJelNuSmFiWGgyWkROTmRsbHVWbkJpUjFGMVpWZEdkQXBpUlVKNVdsZGFla3d5YUd4WlYxSjZUREl4YUdGWE5IZFBRVmxMUzNkWlFrSkJSMFIyZWtGQ1EyZFJjVVJEYUcxYVZFcHBXV3BaTWxwdFZteE9iVTE1Q2xsVVl6TmFWMDVxV21wQk5GbFVRVEZOUkdocldYcFZNMDVFU1RSYVJGRTBUMGRXYTAxQ2MwZERhWE5IUVZGUlFtYzNPSGRCVVhORlJGRjNUR015Vm5NS1dta3hiMkl6VGpCYVYxRjNVVUZaUzB0M1dVSkNRVWRFZG5wQlFrUkJVWGxFUkVKdlpFaFNkMk42YjNaTU1tUndaRWRvTVZscE5XcGlNakIyWVVoV2JncGFNbXgxV2pKYWFGa3lWWFpoTWxaNVltMVdjMk41TVdwaU1qRjBaRmMxY0dSSWEzZFBRVmxMUzNkWlFrSkJSMFIyZWtGQ1JGRlJjVVJEYUcxYVZFcHBDbGxxV1RKYWJWWnNUbTFOZVZsVVl6TmFWMDVxV21wQk5GbFVRVEZOUkdocldYcFZNMDVFU1RSYVJGRTBUMGRXYTAxQ09FZERhWE5IUVZGUlFtYzNPSGNLUVZFMFJVVlJkMUJqYlZadFkzazViMXBYUm10amVUbDBXVmRzZFUxQ2IwZERhWE5IUVZGUlFtYzNPSGRCVVRoRlJFRjNTMDFVUVROTlZGRXpUbFJWZVFwUFZFRjFRbWR2Y2tKblJVVkJXVTh2VFVGRlVVSkRRVTFJYldnd1pFaENlazlwT0haYU1td3dZVWhXYVV4dFRuWmlVemx2WkZka2JtRlhOVzVhYlVacUNscFVRVmxDWjI5eVFtZEZSVUZaVHk5TlFVVlNRa0Z2VFVORVNURk9la2wzVG5wUmVrMUhNRWREYVhOSFFWRlJRbWMzT0hkQlVrbEZXSGQ0WkdGSVVqQUtZMGhOTmt4NU9XNWhXRkp2WkZkSmRWa3lPWFJNTW1neFdqSmtjR0p0WkcxWlYwNXNUREowYkdOdE5XeGlTRTEwV1RJNWRHSllWblZoV0ZJMVRIazFiZ3BoV0ZKdlpGZEpkbVF5T1hsaE1scHpZak5rZWt3eVNqRmhWM2hyVEc1c2FHSlhlRUZqYlZadFkzazViMXBYUm10amVUbDBXVmRzZFUxRVowZERhWE5IQ2tGUlVVSm5OemgzUVZKTlJVdG5kMjlhYlZWNVdXMUpNazV0V214YVZGcHFUVzFGTTA0eVZtcFpNbGwzVDBkRmQwNVVRVFJhUjAweFRucFJlVTlIVVRBS1QwUm9iRnBFUVdoQ1oyOXlRbWRGUlVGWlR5OU5RVVZWUWtKTlRVVllaSFpqYlhSdFlrYzVNMWd5VW5Cak0wSm9aRWRPYjAxSFVVZERhWE5IUVZGUlFncG5OemgzUVZKVlJWWm5lRlZoU0ZJd1kwaE5Oa3g1T1c1aFdGSnZaRmRKZFZreU9YUk1NbWd4V2pKa2NHSnRaRzFaVjA1c1RESjBiR050Tld4aVNFMTBDbGt5T1hSaVdGWjFZVmhTTlV3eVJtcGtSMngyWW01TmRtTnVWblZqZVRoNlQxUkplazlVVlhoUFJFVjNUa001YUdSSVVteGlXRUl3WTNrNGVFMUNXVWNLUTJselIwRlJVVUpuTnpoM1FWSlpSVU5CZDBkalNGWnBZa2RzYWsxRldVZERhWE5IUVZGUlFtYzNPSGRCVW1kRlQwRjNNbU50Vm5kaWVuQnZaRmRrYmdwaFZ6VnVXbTFHYWxwVE9YSmFXRXAxV2xkNGVreFhUblppVnpFeFltMXNNR1ZVY0hsYVYxazJZMjFXYldONU9XOWFWMFpyWTNrNWRGbFhiSFZOU1VkTENrSm5iM0pDWjBWRlFXUmFOVUZuVVVOQ1NIZEZaV2RDTkVGSVdVRXpWREIzWVhOaVNFVlVTbXBIVWpSamJWZGpNMEZ4U2t0WWNtcGxVRXN6TDJnMGNIa0taME00Y0Rkdk5FRkJRVWRvVm10d2FIUm5RVUZDUVUxQlVucENSa0ZwUlVFdlZpOVlVVEY2YjBGcGVUbEtlRFpNZDNOSE1uWTVkR1U0VVUxTlpFdG5id28xYkhoNk5pOURTRlZaV1VOSlJVSmFNbWxrTVRkVU5XRkxLMmRsZUdWV2VVa3ZkVmd4WlVaM2JYaHNOWGRYUW5Ga1UwdHJTRWxtVjAxQmIwZERRM0ZIQ2xOTk5EbENRVTFFUVRKclFVMUhXVU5OVVVOMFExaFZabVowWkU5b1R6Um1lRXdyY0ZoQmVWSnJkRGw1UVU1aGRIY3lUMll4U1RKdldFcGhNazQxSzBrS1p5dE9WbWsxVEV4T2RsWnJTWGQxY2pGa1RVTk5VVVJpZWxWbmFUTTRRVkpPVVhGM05FNUhRMnQxTkhWVGRFNXlNREp4WVVZemFtVkVhbE5YVUdwalNBcG1TbVpHZUhkRVpXSnFTME5KVTNRdlZ6SjFOREpSYnowS0xTMHRMUzFGVGtRZ1EwVlNWRWxHU1VOQlZFVXRMUzB0TFFvPSJ9fX19"}], "timestampVerificationData":{"rfc3161Timestamps":[{"signedTimestamp":"MIICyTADAgEAMIICwAYJKoZIhvcNAQcCoIICsTCCAq0CAQMxDTALBglghkgBZQMEAgEwgbgGCyqGSIb3DQEJEAEEoIGoBIGlMIGiAgEBBgkrBgEEAYO/MAIwMTANBglghkgBZQMEAgEFAAQgmTkV835pgwxkvMNukL50F/XFs1cK4XlYhGVjIf6CtBsCFQCFgrdHkuvKvwpQj8RpZZmBpRIUlhgPMjAyNjEwMTkyMjMxMTdaMAMCAQGgMqQwMC4xFTATBgNVBAoTDHNpZ3N0b3JlLmRldjEVMBMGA1UEAxMMc2lnc3RvcmUtdHNhoAAxggHaMIIB1gIBATBRMDkxFTATBgNVBAoTDHNpZ3N0b3JlLmRldjEgMB4GA1UEAxMXc2lnc3RvcmUtdHNhLXNlbGZzaWduZWQCFDoTVC8MkGHuvMFDL8uKjosqI4sMMAsGCWCGSAFlAwQCAaCB/DAaBgkqhkiG9w0BCQMxDQYLKoZIhvcNAQkQAQQwHAYJKoZIhvcNAQkFMQ8XDTI2MTAxOTIyMzExN1owLwYJKoZIhvcNAQkEMSIEIJRd9HQUlnO59ivgTId0aXPDBwASdkAbCnMh269+NWAGMIGOBgsqhkiG9w0BCRACLzF/MH0wezB5BCCF+Se8B6tiysO0Q1bBDvyBssaIP9p6uebYcNnROs0FtzBVMD2kOzA5MRUwEwYDVQQKEwxzaWdzdG9yZS5kZXYxIDAeBgNVBAMTF3NpZ3N0b3JlLXRzYS1zZWxmc2lnbmVkAhQ6E1QvDJBh7rzBQy/Lio6LKiOLDDAKBggqhkjOPQQDAgRmMGQCMB8mlkizvZTubl6yOTamotJcCj+VOyU8JO51/iJzccSbUZEK7a58vYbacjQtKz0LdQIwPO96KwU6xzAQ+JfLYq0wy1m87aCJg68U22OyZ0sqwJvVatEW1hTkicd7Gh5GBX98"}]}}, "messageSignature":{"messageDigest":{"algorithm":"SHA2_256", "digest":"NXGZG4jSIK0uAlaeJrG0POkgFlQ8WKcaybuxOUHRDEM="}, "signature":"MEUCIDMPLvtPJvAVoCIk/wdcXVImMaCBmmZ4vwsam7XbgHaqAiEApm1p93YygRTpNtLGyHXfqmJq52gglIXqDJxv4LHq37U="}}
build/torch212-rocm72-x86_64-linux/mha.py ADDED
@@ -0,0 +1,332 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FlashAttention forward entry points backed by aiter's Composable-Kernel FMHA.
2
+
3
+ This is the compiled (HIP / Composable Kernel) counterpart to the Triton
4
+ ``aiter-flash-attn`` package. Only the *forward* pass is provided (dense and
5
+ variable-length), including support for learnable attention sinks via
6
+ ``window_size[2]`` / ``s_aux`` (e.g. gpt-oss). The op set ships exclusively
7
+ the CK ``mha_fwd`` / ``mha_varlen_fwd`` kernels, so the sink path is always
8
+ taken (no fall-through to a non-sink ASM kernel).
9
+ """
10
+
11
+ from typing import Optional, Tuple, Union
12
+
13
+ import torch
14
+ from torch import Tensor
15
+
16
+ from ._ops import ops
17
+
18
+
19
+ def _round_multiple(x: int, m: int) -> int:
20
+ return (x + m - 1) // m * m
21
+
22
+
23
+ def _maybe_pad_headdim(t: Tensor, target: int) -> Tensor:
24
+ og = t.size(-1)
25
+ if og == target:
26
+ return t
27
+ return torch.nn.functional.pad(t, [0, target - og])
28
+
29
+
30
+ def _unpack_window(window_size) -> Tuple[int, int, int]:
31
+ left = int(window_size[0])
32
+ right = int(window_size[1])
33
+ sink = int(window_size[2]) if len(window_size) == 3 else 0
34
+ return left, right, sink
35
+
36
+
37
+ def _prep_sink(s_aux: Optional[Tensor], nheads: int, device) -> Optional[Tensor]:
38
+ if s_aux is None:
39
+ return None
40
+ assert s_aux.device == device, "s_aux must be on the same device as q"
41
+ assert s_aux.shape[0] == nheads, "s_aux must have shape [nheads_q]"
42
+ if s_aux.dtype != torch.float32:
43
+ s_aux = s_aux.to(torch.float32)
44
+ return s_aux
45
+
46
+
47
+ def _effective_sink_size(sink_size: int, s_aux: Optional[Tensor]) -> int:
48
+ # The CK kernel only takes the sink path when sink_size > 0. Callers using
49
+ # the transformers attention interface pass the per-head sink logits as
50
+ # ``s_aux`` but leave ``window_size`` a 2-tuple (sink_size == 0), so enable
51
+ # a single learnable sink automatically when s_aux is given.
52
+ if s_aux is not None and sink_size == 0:
53
+ return 1
54
+ return sink_size
55
+
56
+
57
+ def flash_attn_func(
58
+ q: Tensor,
59
+ k: Tensor,
60
+ v: Tensor,
61
+ dropout_p: float = 0.0,
62
+ softmax_scale: Optional[float] = None,
63
+ causal: bool = False,
64
+ window_size: Tuple[int, ...] = (-1, -1, 0),
65
+ bias: Optional[Tensor] = None,
66
+ alibi_slopes: Optional[Tensor] = None,
67
+ return_lse: bool = False,
68
+ return_attn_probs: bool = False,
69
+ s_aux: Optional[Tensor] = None,
70
+ **kwargs,
71
+ ) -> Union[Tensor, Tuple[Tensor, ...]]:
72
+ """Dense FlashAttention forward.
73
+
74
+ ``q, k, v`` have shape ``(batch, seqlen, nheads, headdim)``. ``window_size``
75
+ is ``(left, right, sink_size)``; ``s_aux`` is an optional ``(nheads_q,)``
76
+ fp32 tensor of learnable sink logits. Extra keyword arguments (e.g. those
77
+ passed by the transformers flash-attention interface) are ignored.
78
+ """
79
+ if softmax_scale is None:
80
+ softmax_scale = q.shape[-1] ** (-0.5)
81
+ window_size_left, window_size_right, sink_size = _unpack_window(window_size)
82
+
83
+ nheads_q = q.size(2)
84
+ s_aux = _prep_sink(s_aux, nheads_q, q.device)
85
+ sink_size = _effective_sink_size(sink_size, s_aux)
86
+
87
+ head_q_og = q.size(-1)
88
+ head_v_og = v.size(-1)
89
+ head_q = _round_multiple(head_q_og, 8)
90
+ head_v = _round_multiple(head_v_og, 8)
91
+ q = _maybe_pad_headdim(q, head_q)
92
+ k = _maybe_pad_headdim(k, head_q)
93
+ v = _maybe_pad_headdim(v, head_v)
94
+
95
+ return_softmax = return_attn_probs and dropout_p > 0
96
+ out, softmax_lse, S_dmask, _ = ops.mha_fwd(
97
+ q,
98
+ k,
99
+ v,
100
+ dropout_p,
101
+ softmax_scale,
102
+ causal,
103
+ window_size_left,
104
+ window_size_right,
105
+ sink_size,
106
+ return_lse or return_attn_probs,
107
+ return_softmax,
108
+ None, # cu_seqlens_q
109
+ None, # cu_seqlens_kv
110
+ None, # out
111
+ bias,
112
+ alibi_slopes,
113
+ None, # q_descale
114
+ None, # k_descale
115
+ None, # v_descale
116
+ s_aux,
117
+ None, # gen
118
+ )
119
+
120
+ out = out[..., :head_v_og]
121
+ if not (return_lse or return_attn_probs):
122
+ return out
123
+ result = (out,)
124
+ if return_lse:
125
+ result = result + (softmax_lse,)
126
+ if return_attn_probs:
127
+ result = result + (S_dmask,)
128
+ return result
129
+
130
+
131
+ def flash_attn_varlen_func(
132
+ q: Tensor,
133
+ k: Tensor,
134
+ v: Tensor,
135
+ cu_seqlens_q: Tensor,
136
+ cu_seqlens_k: Tensor,
137
+ max_seqlen_q: int,
138
+ max_seqlen_k: int,
139
+ dropout_p: float = 0.0,
140
+ softmax_scale: Optional[float] = None,
141
+ causal: bool = False,
142
+ window_size: Tuple[int, ...] = (-1, -1, 0),
143
+ bias: Optional[Tensor] = None,
144
+ alibi_slopes: Optional[Tensor] = None,
145
+ return_lse: bool = False,
146
+ return_attn_probs: bool = False,
147
+ block_table: Optional[Tensor] = None,
148
+ min_seqlen_q: int = 0,
149
+ logits_soft_cap: float = 0.0,
150
+ zero_tensors: bool = False,
151
+ cu_seqlens_q_padded: Optional[Tensor] = None,
152
+ cu_seqlens_k_padded: Optional[Tensor] = None,
153
+ s_aux: Optional[Tensor] = None,
154
+ **kwargs,
155
+ ) -> Union[Tensor, Tuple[Tensor, ...]]:
156
+ """Variable-length / packed FlashAttention forward.
157
+
158
+ ``q`` has shape ``(total_q, nheads, headdim)``; ``cu_seqlens_*`` are int32
159
+ cumulative offsets of shape ``(batch + 1,)``. Sinks are supported through
160
+ ``window_size[2]`` / ``s_aux`` as in :func:`flash_attn_func`. Extra keyword
161
+ arguments (e.g. from the transformers flash-attention interface) are ignored.
162
+ """
163
+ if softmax_scale is None:
164
+ softmax_scale = q.shape[-1] ** (-0.5)
165
+ window_size_left, window_size_right, sink_size = _unpack_window(window_size)
166
+
167
+ nheads_q = q.size(1)
168
+ s_aux = _prep_sink(s_aux, nheads_q, q.device)
169
+ sink_size = _effective_sink_size(sink_size, s_aux)
170
+
171
+ head_q_og = q.size(-1)
172
+ head_v_og = v.size(-1)
173
+ head_q = _round_multiple(head_q_og, 8)
174
+ head_v = _round_multiple(head_v_og, 8)
175
+ q = _maybe_pad_headdim(q, head_q)
176
+ k = _maybe_pad_headdim(k, head_q)
177
+ v = _maybe_pad_headdim(v, head_v)
178
+
179
+ return_softmax = return_attn_probs and dropout_p > 0
180
+ out, softmax_lse, S_dmask, _ = ops.mha_varlen_fwd(
181
+ q,
182
+ k,
183
+ v,
184
+ cu_seqlens_q,
185
+ cu_seqlens_k,
186
+ max_seqlen_q,
187
+ max_seqlen_k,
188
+ min_seqlen_q,
189
+ dropout_p,
190
+ softmax_scale,
191
+ logits_soft_cap,
192
+ zero_tensors,
193
+ causal,
194
+ window_size_left,
195
+ window_size_right,
196
+ sink_size,
197
+ return_lse or return_attn_probs,
198
+ return_softmax,
199
+ None, # out
200
+ block_table,
201
+ bias,
202
+ alibi_slopes,
203
+ None, # q_descale
204
+ None, # k_descale
205
+ None, # v_descale
206
+ None, # gen
207
+ cu_seqlens_q_padded,
208
+ cu_seqlens_k_padded,
209
+ s_aux,
210
+ )
211
+
212
+ out = out[..., :head_v_og]
213
+ if not (return_lse or return_attn_probs):
214
+ return out
215
+ result = (out,)
216
+ if return_lse:
217
+ result = result + (softmax_lse,)
218
+ if return_attn_probs:
219
+ result = result + (S_dmask,)
220
+ return result
221
+
222
+
223
+ def _append_to_contiguous_cache(k_cache, v_cache, k, v, cache_seqlens, Sq):
224
+ """In-place append of new k/v into a contiguous cache at per-batch offsets."""
225
+ B = k_cache.shape[0]
226
+ if isinstance(cache_seqlens, int):
227
+ cache_seqlens = torch.full((B,), cache_seqlens, device=k_cache.device, dtype=torch.long)
228
+ pos = cache_seqlens.view(B, 1).to(torch.long) + torch.arange(Sq, device=k_cache.device).view(1, Sq)
229
+ bidx = torch.arange(B, device=k_cache.device).view(B, 1).expand(B, Sq)
230
+ k_cache[bidx, pos] = k.to(k_cache.dtype)
231
+ v_cache[bidx, pos] = v.to(v_cache.dtype)
232
+
233
+
234
+ def flash_attn_with_kvcache(
235
+ q: Tensor,
236
+ k_cache: Tensor,
237
+ v_cache: Tensor,
238
+ k: Optional[Tensor] = None,
239
+ v: Optional[Tensor] = None,
240
+ rotary_cos: Optional[Tensor] = None,
241
+ rotary_sin: Optional[Tensor] = None,
242
+ cache_seqlens=None,
243
+ block_table: Optional[Tensor] = None,
244
+ softmax_scale: Optional[float] = None,
245
+ causal: bool = True,
246
+ window_size: Tuple[int, ...] = (-1, -1, 0),
247
+ s_aux: Optional[Tensor] = None,
248
+ return_lse: bool = False,
249
+ **kwargs,
250
+ ) -> Union[Tensor, Tuple[Tensor, ...]]:
251
+ """FlashAttention against a KV cache, built on the CK split-KV kernel.
252
+
253
+ ``q`` is ``(batch, seqlen_q, nheads, headdim)``. For a contiguous cache,
254
+ ``k_cache``/``v_cache`` are ``(batch, seqlen_cache, nheads_k, headdim)`` and
255
+ ``cache_seqlens`` (int or ``(batch,)``) is the number of valid cache entries
256
+ *before* appending. If ``k``/``v`` (the new tokens) are given, they are
257
+ written into the cache in place. The query attends to the valid prefix
258
+ (``cache_seqlens`` + ``seqlen_q``) with bottom-right causal alignment.
259
+
260
+ Notes / current limitations:
261
+ - Rotary embedding is not applied here; pass ``q``/``k`` already rotated
262
+ (``rotary_cos``/``rotary_sin`` are rejected).
263
+ - Paged caches are addressed via ``block_table`` (forwarded to the split-KV
264
+ varlen kernel); in-cache append for paged layouts must be done by the
265
+ caller (pass ``k=v=None``).
266
+ """
267
+ if rotary_cos is not None or rotary_sin is not None:
268
+ raise NotImplementedError(
269
+ "flash_attn_with_kvcache does not apply rotary; rotate q/k beforehand."
270
+ )
271
+ if softmax_scale is None:
272
+ softmax_scale = q.shape[-1] ** (-0.5)
273
+ window_size_left, window_size_right, sink_size = _unpack_window(window_size)
274
+ B, Sq, Hq, D = q.shape
275
+ s_aux = _prep_sink(s_aux, Hq, q.device)
276
+ sink_size = _effective_sink_size(sink_size, s_aux)
277
+
278
+ paged = block_table is not None
279
+
280
+ # ---- append new tokens into the cache (contiguous layout only) ----
281
+ if k is not None and not paged:
282
+ assert cache_seqlens is not None, "cache_seqlens required to append into the cache"
283
+ _append_to_contiguous_cache(k_cache, v_cache, k, v, cache_seqlens, Sq)
284
+ elif k is not None and paged:
285
+ raise NotImplementedError(
286
+ "in-cache append for paged layouts is not supported; append externally "
287
+ "and call with k=v=None."
288
+ )
289
+
290
+ appended = 0 if k is None else Sq
291
+
292
+ # ---- key lengths ----
293
+ if cache_seqlens is None:
294
+ klens = torch.full((B,), k_cache.shape[1], device=q.device, dtype=torch.long)
295
+ elif isinstance(cache_seqlens, int):
296
+ klens = torch.full((B,), cache_seqlens + appended, device=q.device, dtype=torch.long)
297
+ else:
298
+ klens = cache_seqlens.to(q.device, torch.long) + appended
299
+
300
+ uniform = bool((klens == klens[0]).all().item())
301
+
302
+ # ---- fast path: contiguous cache, uniform length -> dense kernel ----
303
+ if not paged and uniform:
304
+ L = int(klens[0].item())
305
+ return flash_attn_func(
306
+ q, k_cache[:, :L], v_cache[:, :L],
307
+ softmax_scale=softmax_scale, causal=causal,
308
+ window_size=(window_size_left, window_size_right, sink_size),
309
+ s_aux=s_aux, return_lse=return_lse,
310
+ )
311
+
312
+ # ---- general path: variable cache lengths -> split-KV varlen kernel ----
313
+ cu_q = torch.arange(0, (B + 1) * Sq, Sq, device=q.device, dtype=torch.int32)
314
+ cu_k = torch.zeros(B + 1, device=q.device, dtype=torch.int32)
315
+ cu_k[1:] = torch.cumsum(klens, 0).to(torch.int32)
316
+ q_packed = q.reshape(B * Sq, Hq, D)
317
+ if paged:
318
+ k_packed, v_packed = k_cache, v_cache # kernel gathers via block_table
319
+ else:
320
+ k_packed = torch.cat([k_cache[b, : int(klens[b])] for b in range(B)], dim=0)
321
+ v_packed = torch.cat([v_cache[b, : int(klens[b])] for b in range(B)], dim=0)
322
+
323
+ out = flash_attn_varlen_func(
324
+ q_packed, k_packed, v_packed, cu_q, cu_k, Sq, int(klens.max().item()),
325
+ softmax_scale=softmax_scale, causal=causal,
326
+ window_size=(window_size_left, window_size_right, sink_size),
327
+ block_table=block_table, s_aux=s_aux, return_lse=return_lse,
328
+ )
329
+ if return_lse:
330
+ out, lse = out
331
+ return out.reshape(B, Sq, Hq, D), lse
332
+ return out.reshape(B, Sq, Hq, D)
build/torch212-rocm72-x86_64-linux/symbols.json ADDED
@@ -0,0 +1,349 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "schema_version": 1,
3
+ "module": "_aiter_flash_attn_ck_rocm_fe2bb66",
4
+ "functions": [
5
+ {
6
+ "name": "flash_attn_func",
7
+ "qualname": "flash_attn_func",
8
+ "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha",
9
+ "kind": "function",
10
+ "doc": "Dense FlashAttention forward.\n\n``q, k, v`` have shape ``(batch, seqlen, nheads, headdim)``. ``window_size``\nis ``(left, right, sink_size)``; ``s_aux`` is an optional ``(nheads_q,)``\nfp32 tensor of learnable sink logits. Extra keyword arguments (e.g. those\npassed by the transformers flash-attention interface) are ignored.",
11
+ "signature": {
12
+ "parameters": [
13
+ {
14
+ "name": "q",
15
+ "kind": "POSITIONAL_OR_KEYWORD",
16
+ "default": null,
17
+ "annotation": "torch.Tensor"
18
+ },
19
+ {
20
+ "name": "k",
21
+ "kind": "POSITIONAL_OR_KEYWORD",
22
+ "default": null,
23
+ "annotation": "torch.Tensor"
24
+ },
25
+ {
26
+ "name": "v",
27
+ "kind": "POSITIONAL_OR_KEYWORD",
28
+ "default": null,
29
+ "annotation": "torch.Tensor"
30
+ },
31
+ {
32
+ "name": "dropout_p",
33
+ "kind": "POSITIONAL_OR_KEYWORD",
34
+ "default": "0.0",
35
+ "annotation": "float"
36
+ },
37
+ {
38
+ "name": "softmax_scale",
39
+ "kind": "POSITIONAL_OR_KEYWORD",
40
+ "default": "None",
41
+ "annotation": "float | None"
42
+ },
43
+ {
44
+ "name": "causal",
45
+ "kind": "POSITIONAL_OR_KEYWORD",
46
+ "default": "False",
47
+ "annotation": "bool"
48
+ },
49
+ {
50
+ "name": "window_size",
51
+ "kind": "POSITIONAL_OR_KEYWORD",
52
+ "default": "(-1, -1, 0)",
53
+ "annotation": "Tuple[int, ...]"
54
+ },
55
+ {
56
+ "name": "bias",
57
+ "kind": "POSITIONAL_OR_KEYWORD",
58
+ "default": "None",
59
+ "annotation": "torch.Tensor | None"
60
+ },
61
+ {
62
+ "name": "alibi_slopes",
63
+ "kind": "POSITIONAL_OR_KEYWORD",
64
+ "default": "None",
65
+ "annotation": "torch.Tensor | None"
66
+ },
67
+ {
68
+ "name": "return_lse",
69
+ "kind": "POSITIONAL_OR_KEYWORD",
70
+ "default": "False",
71
+ "annotation": "bool"
72
+ },
73
+ {
74
+ "name": "return_attn_probs",
75
+ "kind": "POSITIONAL_OR_KEYWORD",
76
+ "default": "False",
77
+ "annotation": "bool"
78
+ },
79
+ {
80
+ "name": "s_aux",
81
+ "kind": "POSITIONAL_OR_KEYWORD",
82
+ "default": "None",
83
+ "annotation": "torch.Tensor | None"
84
+ },
85
+ {
86
+ "name": "kwargs",
87
+ "kind": "VAR_KEYWORD",
88
+ "default": null,
89
+ "annotation": null
90
+ }
91
+ ],
92
+ "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]"
93
+ }
94
+ },
95
+ {
96
+ "name": "flash_attn_varlen_func",
97
+ "qualname": "flash_attn_varlen_func",
98
+ "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha",
99
+ "kind": "function",
100
+ "doc": "Variable-length / packed FlashAttention forward.\n\n``q`` has shape ``(total_q, nheads, headdim)``; ``cu_seqlens_*`` are int32\ncumulative offsets of shape ``(batch + 1,)``. Sinks are supported through\n``window_size[2]`` / ``s_aux`` as in :func:`flash_attn_func`. Extra keyword\narguments (e.g. from the transformers flash-attention interface) are ignored.",
101
+ "signature": {
102
+ "parameters": [
103
+ {
104
+ "name": "q",
105
+ "kind": "POSITIONAL_OR_KEYWORD",
106
+ "default": null,
107
+ "annotation": "torch.Tensor"
108
+ },
109
+ {
110
+ "name": "k",
111
+ "kind": "POSITIONAL_OR_KEYWORD",
112
+ "default": null,
113
+ "annotation": "torch.Tensor"
114
+ },
115
+ {
116
+ "name": "v",
117
+ "kind": "POSITIONAL_OR_KEYWORD",
118
+ "default": null,
119
+ "annotation": "torch.Tensor"
120
+ },
121
+ {
122
+ "name": "cu_seqlens_q",
123
+ "kind": "POSITIONAL_OR_KEYWORD",
124
+ "default": null,
125
+ "annotation": "torch.Tensor"
126
+ },
127
+ {
128
+ "name": "cu_seqlens_k",
129
+ "kind": "POSITIONAL_OR_KEYWORD",
130
+ "default": null,
131
+ "annotation": "torch.Tensor"
132
+ },
133
+ {
134
+ "name": "max_seqlen_q",
135
+ "kind": "POSITIONAL_OR_KEYWORD",
136
+ "default": null,
137
+ "annotation": "int"
138
+ },
139
+ {
140
+ "name": "max_seqlen_k",
141
+ "kind": "POSITIONAL_OR_KEYWORD",
142
+ "default": null,
143
+ "annotation": "int"
144
+ },
145
+ {
146
+ "name": "dropout_p",
147
+ "kind": "POSITIONAL_OR_KEYWORD",
148
+ "default": "0.0",
149
+ "annotation": "float"
150
+ },
151
+ {
152
+ "name": "softmax_scale",
153
+ "kind": "POSITIONAL_OR_KEYWORD",
154
+ "default": "None",
155
+ "annotation": "float | None"
156
+ },
157
+ {
158
+ "name": "causal",
159
+ "kind": "POSITIONAL_OR_KEYWORD",
160
+ "default": "False",
161
+ "annotation": "bool"
162
+ },
163
+ {
164
+ "name": "window_size",
165
+ "kind": "POSITIONAL_OR_KEYWORD",
166
+ "default": "(-1, -1, 0)",
167
+ "annotation": "Tuple[int, ...]"
168
+ },
169
+ {
170
+ "name": "bias",
171
+ "kind": "POSITIONAL_OR_KEYWORD",
172
+ "default": "None",
173
+ "annotation": "torch.Tensor | None"
174
+ },
175
+ {
176
+ "name": "alibi_slopes",
177
+ "kind": "POSITIONAL_OR_KEYWORD",
178
+ "default": "None",
179
+ "annotation": "torch.Tensor | None"
180
+ },
181
+ {
182
+ "name": "return_lse",
183
+ "kind": "POSITIONAL_OR_KEYWORD",
184
+ "default": "False",
185
+ "annotation": "bool"
186
+ },
187
+ {
188
+ "name": "return_attn_probs",
189
+ "kind": "POSITIONAL_OR_KEYWORD",
190
+ "default": "False",
191
+ "annotation": "bool"
192
+ },
193
+ {
194
+ "name": "block_table",
195
+ "kind": "POSITIONAL_OR_KEYWORD",
196
+ "default": "None",
197
+ "annotation": "torch.Tensor | None"
198
+ },
199
+ {
200
+ "name": "min_seqlen_q",
201
+ "kind": "POSITIONAL_OR_KEYWORD",
202
+ "default": "0",
203
+ "annotation": "int"
204
+ },
205
+ {
206
+ "name": "logits_soft_cap",
207
+ "kind": "POSITIONAL_OR_KEYWORD",
208
+ "default": "0.0",
209
+ "annotation": "float"
210
+ },
211
+ {
212
+ "name": "zero_tensors",
213
+ "kind": "POSITIONAL_OR_KEYWORD",
214
+ "default": "False",
215
+ "annotation": "bool"
216
+ },
217
+ {
218
+ "name": "cu_seqlens_q_padded",
219
+ "kind": "POSITIONAL_OR_KEYWORD",
220
+ "default": "None",
221
+ "annotation": "torch.Tensor | None"
222
+ },
223
+ {
224
+ "name": "cu_seqlens_k_padded",
225
+ "kind": "POSITIONAL_OR_KEYWORD",
226
+ "default": "None",
227
+ "annotation": "torch.Tensor | None"
228
+ },
229
+ {
230
+ "name": "s_aux",
231
+ "kind": "POSITIONAL_OR_KEYWORD",
232
+ "default": "None",
233
+ "annotation": "torch.Tensor | None"
234
+ },
235
+ {
236
+ "name": "kwargs",
237
+ "kind": "VAR_KEYWORD",
238
+ "default": null,
239
+ "annotation": null
240
+ }
241
+ ],
242
+ "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]"
243
+ }
244
+ },
245
+ {
246
+ "name": "flash_attn_with_kvcache",
247
+ "qualname": "flash_attn_with_kvcache",
248
+ "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha",
249
+ "kind": "function",
250
+ "doc": "FlashAttention against a KV cache, built on the CK split-KV kernel.\n\n``q`` is ``(batch, seqlen_q, nheads, headdim)``. For a contiguous cache,\n``k_cache``/``v_cache`` are ``(batch, seqlen_cache, nheads_k, headdim)`` and\n``cache_seqlens`` (int or ``(batch,)``) is the number of valid cache entries\n*before* appending. If ``k``/``v`` (the new tokens) are given, they are\nwritten into the cache in place. The query attends to the valid prefix\n(``cache_seqlens`` + ``seqlen_q``) with bottom-right causal alignment.\n\nNotes / current limitations:\n- Rotary embedding is not applied here; pass ``q``/``k`` already rotated\n (``rotary_cos``/``rotary_sin`` are rejected).\n- Paged caches are addressed via ``block_table`` (forwarded to the split-KV\n varlen kernel); in-cache append for paged layouts must be done by the\n caller (pass ``k=v=None``).",
251
+ "signature": {
252
+ "parameters": [
253
+ {
254
+ "name": "q",
255
+ "kind": "POSITIONAL_OR_KEYWORD",
256
+ "default": null,
257
+ "annotation": "torch.Tensor"
258
+ },
259
+ {
260
+ "name": "k_cache",
261
+ "kind": "POSITIONAL_OR_KEYWORD",
262
+ "default": null,
263
+ "annotation": "torch.Tensor"
264
+ },
265
+ {
266
+ "name": "v_cache",
267
+ "kind": "POSITIONAL_OR_KEYWORD",
268
+ "default": null,
269
+ "annotation": "torch.Tensor"
270
+ },
271
+ {
272
+ "name": "k",
273
+ "kind": "POSITIONAL_OR_KEYWORD",
274
+ "default": "None",
275
+ "annotation": "torch.Tensor | None"
276
+ },
277
+ {
278
+ "name": "v",
279
+ "kind": "POSITIONAL_OR_KEYWORD",
280
+ "default": "None",
281
+ "annotation": "torch.Tensor | None"
282
+ },
283
+ {
284
+ "name": "rotary_cos",
285
+ "kind": "POSITIONAL_OR_KEYWORD",
286
+ "default": "None",
287
+ "annotation": "torch.Tensor | None"
288
+ },
289
+ {
290
+ "name": "rotary_sin",
291
+ "kind": "POSITIONAL_OR_KEYWORD",
292
+ "default": "None",
293
+ "annotation": "torch.Tensor | None"
294
+ },
295
+ {
296
+ "name": "cache_seqlens",
297
+ "kind": "POSITIONAL_OR_KEYWORD",
298
+ "default": "None",
299
+ "annotation": null
300
+ },
301
+ {
302
+ "name": "block_table",
303
+ "kind": "POSITIONAL_OR_KEYWORD",
304
+ "default": "None",
305
+ "annotation": "torch.Tensor | None"
306
+ },
307
+ {
308
+ "name": "softmax_scale",
309
+ "kind": "POSITIONAL_OR_KEYWORD",
310
+ "default": "None",
311
+ "annotation": "float | None"
312
+ },
313
+ {
314
+ "name": "causal",
315
+ "kind": "POSITIONAL_OR_KEYWORD",
316
+ "default": "True",
317
+ "annotation": "bool"
318
+ },
319
+ {
320
+ "name": "window_size",
321
+ "kind": "POSITIONAL_OR_KEYWORD",
322
+ "default": "(-1, -1, 0)",
323
+ "annotation": "Tuple[int, ...]"
324
+ },
325
+ {
326
+ "name": "s_aux",
327
+ "kind": "POSITIONAL_OR_KEYWORD",
328
+ "default": "None",
329
+ "annotation": "torch.Tensor | None"
330
+ },
331
+ {
332
+ "name": "return_lse",
333
+ "kind": "POSITIONAL_OR_KEYWORD",
334
+ "default": "False",
335
+ "annotation": "bool"
336
+ },
337
+ {
338
+ "name": "kwargs",
339
+ "kind": "VAR_KEYWORD",
340
+ "default": null,
341
+ "annotation": null
342
+ }
343
+ ],
344
+ "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]"
345
+ }
346
+ }
347
+ ],
348
+ "layers": []
349
+ }
build/torch213-rocm71-x86_64-linux/__init__.py ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """AITER Composable-Kernel FlashAttention kernels for AMD ROCm (compiled HIP).
2
+
3
+ Compiled (Composable Kernel / HIP) counterpart to the Triton-based
4
+ ``aiter-flash-attn`` package, repackaged from the CK FMHA forward path of the
5
+ `ROCm/aiter <https://github.com/ROCm/aiter>`_ project as a self-contained
6
+ Hugging Face Hub kernel. Exposes the FlashAttention forward entry points with
7
+ learnable attention sink support (e.g. gpt-oss).
8
+ """
9
+
10
+ from .mha import (
11
+ flash_attn_func,
12
+ flash_attn_varlen_func,
13
+ flash_attn_with_kvcache,
14
+ )
15
+
16
+
17
+ __kernel_metadata__ = {
18
+ "license": "mit",
19
+ }
20
+
21
+
22
+ __all__ = [
23
+ "__kernel_metadata__",
24
+ "flash_attn_func",
25
+ "flash_attn_varlen_func",
26
+ "flash_attn_with_kvcache",
27
+ ]
build/torch213-rocm71-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2a171f33a155682ab9672636dfb2f60fa988da30f3d8a9f6fff3d48fb6032ea6
3
+ size 201962776
build/torch213-rocm71-x86_64-linux/_ops.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _aiter_flash_attn_ck_rocm_fe2bb66
3
+ ops = torch.ops._aiter_flash_attn_ck_rocm_fe2bb66
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ """
7
+ Prefix op by namespace.
8
+ """
9
+ return f"_aiter_flash_attn_ck_rocm_fe2bb66::{op_name}"
build/torch213-rocm71-x86_64-linux/metadata.json ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "aiter-flash-attn-ck",
3
+ "id": "_aiter_flash_attn_ck_rocm_fe2bb66",
4
+ "version": 1,
5
+ "kernels-minver": "0.14.0",
6
+ "license": "MIT",
7
+ "python-depends": [],
8
+ "kernel-depends": [],
9
+ "backend": {
10
+ "type": "rocm",
11
+ "archs": [
12
+ "gfx942",
13
+ "gfx950"
14
+ ]
15
+ },
16
+ "digest": {
17
+ "algorithm": "sha256",
18
+ "files": {
19
+ "__init__.py": "7wj1GTm2bqTLLCyT4xLuAk/+JWYrhINjY/ODMNWhgZ0=",
20
+ "_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so": "KhcfM6FVaCq5ZyY237L2D6mI2jDz2Kn2//PUj7YDLqY=",
21
+ "_ops.py": "4WkdYDWWJp60zep+oW/5ym9vIeVMNfrgRCkP3o/w7yU=",
22
+ "mha.py": "zCAjY4O+DiCzmEL+W9sqXxZaqjsj4RJbWFcyPpUZejk=",
23
+ "symbols.json": "rhnF4H8Jsb6I+YPxIVFMgnLcTQJODEXbZ7ge77NZXAU="
24
+ }
25
+ },
26
+ "provenance": {
27
+ "kernel-builder": {
28
+ "version": "0.18.0-dev0",
29
+ "commit": "d054906a13710b811b96697b4635f03020842733",
30
+ "dirty": false
31
+ },
32
+ "kernel": {
33
+ "commit": "fe2bb66fee6c2a77eccf08a0508dc57428d488ed",
34
+ "dirty": false
35
+ }
36
+ }
37
+ }
build/torch213-rocm71-x86_64-linux/metadata.json.sigstore ADDED
@@ -0,0 +1 @@
 
 
1
+ {"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json", "verificationMaterial":{"certificate":{"rawBytes":"MIIHLTCCBrKgAwIBAgIUYhDBz3s6J742wscabPTQJuZF0E4wCgYIKoZIzj0EAwMwNzEVMBMGA1UEChMMc2lnc3RvcmUuZGV2MR4wHAYDVQQDExVzaWdzdG9yZS1pbnRlcm1lZGlhdGUwHhcNMjYxMDE5MjIzMTE4WhcNMjYxMDE5MjI0MTE4WjAAMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEp3g/dI77cg1tnq5eu54MWYrJ8cbb8SzEVubcy/U67+Aiwr4Y0MWc384cRXpKZj0FhdoGxrrETLe96k9d+zBzgaOCBdEwggXNMA4GA1UdDwEB/wQEAwIHgDATBgNVHSUEDDAKBggrBgEFBQcDAzAfBgNVHSMEGDAWgBTf0+nPViQRlvmo2OkoVaLGLhhkPzBrBgNVHREBAf8EYTBfhl1odHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkvLmdpdGh1Yi93b3JrZmxvd3MvYnVpbGQueWFtbEByZWZzL2hlYWRzL21haW4wOQYKKwYBBAGDvzABAQQraHR0cHM6Ly90b2tlbi5hY3Rpb25zLmdpdGh1YnVzZXJjb250ZW50LmNvbTAfBgorBgEEAYO/MAECBBF3b3JrZmxvd19kaXNwYXRjaDA2BgorBgEEAYO/MAEDBChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMBMGCisGAQQBg78wAQQEBUJ1aWxkMCsGCisGAQQBg78wAQUEHWh1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5MB0GCisGAQQBg78wAQYED3JlZnMvaGVhZHMvbWFpbjA7BgorBgEEAYO/MAEIBC0MK2h0dHBzOi8vdG9rZW4uYWN0aW9ucy5naXRodWJ1c2VyY29udGVudC5jb20wbQYKKwYBBAGDvzABCQRfDF1odHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkvLmdpdGh1Yi93b3JrZmxvd3MvYnVpbGQueWFtbEByZWZzL2hlYWRzL21haW4wOAYKKwYBBAGDvzABCgQqDChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMBsGCisGAQQBg78wAQsEDQwLc2VsZi1ob3N0ZWQwQAYKKwYBBAGDvzABDAQyDDBodHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkwOAYKKwYBBAGDvzABDQQqDChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMB8GCisGAQQBg78wAQ4EEQwPcmVmcy9oZWFkcy9tYWluMBoGCisGAQQBg78wAQ8EDAwKMTA3MTQ3NTUyOTAuBgorBgEEAYO/MAEQBCAMHmh0dHBzOi8vZ2l0aHViLmNvbS9odWdnaW5nZmFjZTAYBgorBgEEAYO/MAERBAoMCDI1NzIwNzQzMG0GCisGAQQBg78wARIEXwxdaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5Ly5naXRodWIvd29ya2Zsb3dzL2J1aWxkLnlhbWxAcmVmcy9oZWFkcy9tYWluMDgGCisGAQQBg78wARMEKgwoZmUyYmI2NmZlZTZjMmE3N2VjY2YwOGEwNTA4ZGM1NzQyOGQ0ODhlZDAhBgorBgEEAYO/MAEUBBMMEXdvcmtmbG93X2Rpc3BhdGNoMGQGCisGAQQBg78wARUEVgxUaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5L2FjdGlvbnMvcnVucy8zOTIzOTUxODEwNC9hdHRlbXB0cy8xMBYGCisGAQQBg78wARYECAwGcHVibGljMEYGCisGAQQBg78wARgEOAw2cmVwbzpodWdnaW5nZmFjZS9rZXJuZWxzLWNvbW11bml0eTpyZWY6cmVmcy9oZWFkcy9tYWluMIGKBgorBgEEAdZ5AgQCBHwEegB4AHYA3T0wasbHETJjGR4cmWc3AqJKXrjePK3/h4pygC8p7o4AAAGhVkplLwAABAMARzBFAiAYzcmarOFIpCdMHLsWMMcmiX3uiDqTtrSbhnugLl9KbQIhAJimKU6Gp6Muq/12zl25mMiXo7YzY9oPcsSkZyGlyJZwMAoGCCqGSM49BAMDA2kAMGYCMQD8NasCHiPCLAtolVO5/0rl5UfBymvjywi0NVKAJdcAvP61/IJlRbX5IZbatI+8YOsCMQCGhXN2BT6CTuaWC3qwLFzACyTV4uinQUnHOzEDc74z1+uGTmHfVTRDIi2/tFQuuNM="}, "tlogEntries":[{"logIndex":"3327212209", "logId":{"keyId":"wNI9atQGlz+VWfO6LRygH4QUfY/8W4RFwiT5i5WRgB0="}, "kindVersion":{"kind":"hashedrekord", "version":"0.0.1"}, "integratedTime":"1792449078", "inclusionPromise":{"signedEntryTimestamp":"MEQCIBsAQmhJQzBVzalYjG0DrFcgZBeNwlfSjSgAmWKG0sPLAiAg/J0Hzy6ImkWvWbDx3/jTT3+cRJDBm2PIrIDTQTwEuw=="}, "inclusionProof":{"logIndex":"3205307947", "rootHash":"W+fxNQGQRlQR8NP5epr4cduN8DmMRMB26mm1Ob1vTrs=", "treeSize":"3205307963", "hashes":["S7xwjCFRfoN65j+YWkYcaKfA6xnjEY+JNyw73H3LYfg=", "1ZB+altVOQMte65LrxRL5bTxYbBz43joc2eTBAALLpA=", "kBfrXfa6yvdyCvkB8KxtuAweraT6oqFH4L025R6tbjk=", "5AYjk/jc91WxLoeKt6AMwJV3depI3V6Yi1pPZMjQMJw=", "iFo9RWcriE709SPAUnZ3E60KJTBscfs+3bxe9WWL2OA=", "pe8st8CnUIruORVX9JaZ2kalqSjxb7oo2wfilP83+bk=", "gBGf/lDkk/5tDR8CLTuE5Pt2YTL1zjLs12T3vvxq7X4=", "0aSE/O6jCq82vSwoybvpblRnnP72+1gEInKy6Wyc3E0=", "5egUlZ4Pi0Ix/qLyCDXocsWywJCHqrde/S06vOZvlOM=", "QsXufUpoKtSjKAB19K1G00umQpWvPHlBNuRGFiiVjQQ=", "QIMkwQbwAewhLJd//5FFwI3SnyNQBGjCgJrelfG8vhc=", "Et17CACUv9QN64irUJuBjsrgRGkGnnJ9xacRV/aBdwQ=", "lqgEwkxhDMs/gdIAE4lmjfMwg7nDmv7lda6O0LGliCw=", "HovTPBWzoErvUg1u5sIi3x1ZonZ7yJekVsBrmpomCIA=", "G+nk7mgE5h6dUmv2UY7NDJrY3WvrZY31raaO1AETt+o=", "qlTWNDzERnTBQVcHHXXVPjiUP0SsikGxfcxVlgqVFDo=", "gg9GNApNke3+zNhl4tsWtmIycCFrOMCB3lys6XuSQNQ=", "NlOclldmKPA1tMKIskqHAwuQDYHp31YwpK+OOMrMccc=", "qxzHanAzz57SDdmJe0B7bJK72NTIBbwEMGBBKvDOROw=", "xH/DCseLHr9eKoYT8qsORZK7zVdEGYWHuVtsVrD95wY="], "checkpoint":{"envelope":"rekor.sigstore.dev - 1193050959916656506\n3205307963\nW+fxNQGQRlQR8NP5epr4cduN8DmMRMB26mm1Ob1vTrs=\n\n— rekor.sigstore.dev wNI9ajBFAiEAxuRsGa2QG9aIT9+2qqvRf3gfyS1DbDaXilVScnJohYUCIDNWE9xwg2HhHzFKbgG8xMK0LNtqkdJC9u2OLYFMDY6H\n"}}, "canonicalizedBody":"eyJhcGlWZXJzaW9uIjoiMC4wLjEiLCJraW5kIjoiaGFzaGVkcmVrb3JkIiwic3BlYyI6eyJkYXRhIjp7Imhhc2giOnsiYWxnb3JpdGhtIjoic2hhMjU2IiwidmFsdWUiOiIwMDhiYTkyYTNlY2M5MjA5ZDBiMDUxOTllZGEzY2IxYmE0ZWM4OGMzZTVmOGI0YWZhYzAwZmIyMTk4OTJkYjRjIn19LCJzaWduYXR1cmUiOnsiY29udGVudCI6Ik1FUUNIMXBXbzdsRVJEbXhFZlk3emZnMlBscjJTb2xqZWR1azZCZGhqYU1samZBQ0lRQzFaZks4Y3VFVERodW5rT0R0MTg3TTJwdlRpOEVWczg0WEwvNDY3RGtXbVE9PSIsInB1YmxpY0tleSI6eyJjb250ZW50IjoiTFMwdExTMUNSVWRKVGlCRFJWSlVTVVpKUTBGVVJTMHRMUzB0Q2sxSlNVaE1WRU5EUW5KTFowRjNTVUpCWjBsVldXaEVRbm96Y3paS056UXlkM05qWVdKUVZGRktkVnBHTUVVMGQwTm5XVWxMYjFwSmVtb3dSVUYzVFhjS1RucEZWazFDVFVkQk1WVkZRMmhOVFdNeWJHNWpNMUoyWTIxVmRWcEhWakpOVWpSM1NFRlpSRlpSVVVSRmVGWjZZVmRrZW1SSE9YbGFVekZ3WW01U2JBcGpiVEZzV2tkc2FHUkhWWGRJYUdOT1RXcFplRTFFUlRWTmFrbDZUVlJGTkZkb1kwNU5hbGw0VFVSRk5VMXFTVEJOVkVVMFYycEJRVTFHYTNkRmQxbElDa3R2V2tsNmFqQkRRVkZaU1V0dldrbDZhakJFUVZGalJGRm5RVVZ3TTJjdlpFazNOMk5uTVhSdWNUVmxkVFUwVFZkWmNrbzRZMkppT0ZONlJWWjFZbU1LZVM5Vk5qY3JRV2wzY2pSWk1FMVhZek00TkdOU1dIQkxXbW93Um1oa2IwZDRjbkpGVkV4bE9UWnJPV1FyZWtKNloyRlBRMEprUlhkbloxaE9UVUUwUndwQk1WVmtSSGRGUWk5M1VVVkJkMGxJWjBSQlZFSm5UbFpJVTFWRlJFUkJTMEpuWjNKQ1owVkdRbEZqUkVGNlFXWkNaMDVXU0ZOTlJVZEVRVmRuUWxSbUNqQXJibEJXYVZGU2JIWnRiekpQYTI5V1lVeEhUR2hvYTFCNlFuSkNaMDVXU0ZKRlFrRm1PRVZaVkVKbWFHd3hiMlJJVW5kamVtOTJUREprY0dSSGFERUtXV2sxYW1JeU1IWmhTRlp1V2pKc2RWb3lXbWhaTWxWMllUSldlV0p0Vm5OamVURnFZakl4ZEdSWE5YQmtTR3QyVEcxa2NHUkhhREZaYVRrellqTktjZ3BhYlhoMlpETk5kbGx1Vm5CaVIxRjFaVmRHZEdKRlFubGFWMXA2VERKb2JGbFhVbnBNTWpGb1lWYzBkMDlSV1V0TGQxbENRa0ZIUkhaNlFVSkJVVkZ5Q21GSVVqQmpTRTAyVEhrNU1HSXlkR3hpYVRWb1dUTlNjR0l5TlhwTWJXUndaRWRvTVZsdVZucGFXRXBxWWpJMU1GcFhOVEJNYlU1MllsUkJaa0puYjNJS1FtZEZSVUZaVHk5TlFVVkRRa0pHTTJJelNuSmFiWGgyWkRFNWEyRllUbmRaV0ZKcVlVUkJNa0puYjNKQ1owVkZRVmxQTDAxQlJVUkNRMmh0V2xSS2FRcFphbGt5V20xV2JFNXRUWGxaVkdNeldsZE9hbHBxUVRSWlZFRXhUVVJvYTFsNlZUTk9SRWswV2tSUk5FOUhWbXROUWsxSFEybHpSMEZSVVVKbk56aDNDa0ZSVVVWQ1ZVb3hZVmQ0YTAxRGMwZERhWE5IUVZGUlFtYzNPSGRCVVZWRlNGZG9NVm95WkhCaWJXUnRXVmRPYkV3eWRHeGpiVFZzWWtoTmRGa3lPWFFLWWxoV2RXRllValZOUWpCSFEybHpSMEZSVVVKbk56aDNRVkZaUlVRelNteGFiazEyWVVkV2FGcElUWFppVjBad1ltcEJOMEpuYjNKQ1owVkZRVmxQTHdwTlFVVkpRa013VFVzeWFEQmtTRUo2VDJrNGRtUkhPWEphVnpSMVdWZE9NR0ZYT1hWamVUVnVZVmhTYjJSWFNqRmpNbFo1V1RJNWRXUkhWblZrUXpWcUNtSXlNSGRpVVZsTFMzZFpRa0pCUjBSMmVrRkNRMUZTWmtSR01XOWtTRkozWTNwdmRrd3laSEJrUjJneFdXazFhbUl5TUhaaFNGWnVXakpzZFZveVdtZ0tXVEpWZG1FeVZubGliVlp6WTNreGFtSXlNWFJrVnpWd1pFaHJka3h0WkhCa1IyZ3hXV2s1TTJJelNuSmFiWGgyWkROTmRsbHVWbkJpUjFGMVpWZEdkQXBpUlVKNVdsZGFla3d5YUd4WlYxSjZUREl4YUdGWE5IZFBRVmxMUzNkWlFrSkJSMFIyZWtGQ1EyZFJjVVJEYUcxYVZFcHBXV3BaTWxwdFZteE9iVTE1Q2xsVVl6TmFWMDVxV21wQk5GbFVRVEZOUkdocldYcFZNMDVFU1RSYVJGRTBUMGRXYTAxQ2MwZERhWE5IUVZGUlFtYzNPSGRCVVhORlJGRjNUR015Vm5NS1dta3hiMkl6VGpCYVYxRjNVVUZaUzB0M1dVSkNRVWRFZG5wQlFrUkJVWGxFUkVKdlpFaFNkMk42YjNaTU1tUndaRWRvTVZscE5XcGlNakIyWVVoV2JncGFNbXgxV2pKYWFGa3lWWFpoTWxaNVltMVdjMk41TVdwaU1qRjBaRmMxY0dSSWEzZFBRVmxMUzNkWlFrSkJSMFIyZWtGQ1JGRlJjVVJEYUcxYVZFcHBDbGxxV1RKYWJWWnNUbTFOZVZsVVl6TmFWMDVxV21wQk5GbFVRVEZOUkdocldYcFZNMDVFU1RSYVJGRTBUMGRXYTAxQ09FZERhWE5IUVZGUlFtYzNPSGNLUVZFMFJVVlJkMUJqYlZadFkzazViMXBYUm10amVUbDBXVmRzZFUxQ2IwZERhWE5IUVZGUlFtYzNPSGRCVVRoRlJFRjNTMDFVUVROTlZGRXpUbFJWZVFwUFZFRjFRbWR2Y2tKblJVVkJXVTh2VFVGRlVVSkRRVTFJYldnd1pFaENlazlwT0haYU1td3dZVWhXYVV4dFRuWmlVemx2WkZka2JtRlhOVzVhYlVacUNscFVRVmxDWjI5eVFtZEZSVUZaVHk5TlFVVlNRa0Z2VFVORVNURk9la2wzVG5wUmVrMUhNRWREYVhOSFFWRlJRbWMzT0hkQlVrbEZXSGQ0WkdGSVVqQUtZMGhOTmt4NU9XNWhXRkp2WkZkSmRWa3lPWFJNTW1neFdqSmtjR0p0WkcxWlYwNXNUREowYkdOdE5XeGlTRTEwV1RJNWRHSllWblZoV0ZJMVRIazFiZ3BoV0ZKdlpGZEpkbVF5T1hsaE1scHpZak5rZWt3eVNqRmhWM2hyVEc1c2FHSlhlRUZqYlZadFkzazViMXBYUm10amVUbDBXVmRzZFUxRVowZERhWE5IQ2tGUlVVSm5OemgzUVZKTlJVdG5kMjlhYlZWNVdXMUpNazV0V214YVZGcHFUVzFGTTA0eVZtcFpNbGwzVDBkRmQwNVVRVFJhUjAweFRucFJlVTlIVVRBS1QwUm9iRnBFUVdoQ1oyOXlRbWRGUlVGWlR5OU5RVVZWUWtKTlRVVllaSFpqYlhSdFlrYzVNMWd5VW5Cak0wSm9aRWRPYjAxSFVVZERhWE5IUVZGUlFncG5OemgzUVZKVlJWWm5lRlZoU0ZJd1kwaE5Oa3g1T1c1aFdGSnZaRmRKZFZreU9YUk1NbWd4V2pKa2NHSnRaRzFaVjA1c1RESjBiR050Tld4aVNFMTBDbGt5T1hSaVdGWjFZVmhTTlV3eVJtcGtSMngyWW01TmRtTnVWblZqZVRoNlQxUkplazlVVlhoUFJFVjNUa001YUdSSVVteGlXRUl3WTNrNGVFMUNXVWNLUTJselIwRlJVVUpuTnpoM1FWSlpSVU5CZDBkalNGWnBZa2RzYWsxRldVZERhWE5IUVZGUlFtYzNPSGRCVW1kRlQwRjNNbU50Vm5kaWVuQnZaRmRrYmdwaFZ6VnVXbTFHYWxwVE9YSmFXRXAxV2xkNGVreFhUblppVnpFeFltMXNNR1ZVY0hsYVYxazJZMjFXYldONU9XOWFWMFpyWTNrNWRGbFhiSFZOU1VkTENrSm5iM0pDWjBWRlFXUmFOVUZuVVVOQ1NIZEZaV2RDTkVGSVdVRXpWREIzWVhOaVNFVlVTbXBIVWpSamJWZGpNMEZ4U2t0WWNtcGxVRXN6TDJnMGNIa0taME00Y0Rkdk5FRkJRVWRvVm10d2JFeDNRVUZDUVUxQlVucENSa0ZwUVZsNlkyMWhjazlHU1hCRFpFMUlUSE5YVFUxamJXbFlNM1ZwUkhGVWRISlRZZ3BvYm5WblRHdzVTMkpSU1doQlNtbHRTMVUyUjNBMlRYVnhMekV5ZW13eU5XMU5hVmh2TjFsNldUbHZVR056VTJ0YWVVZHNlVXBhZDAxQmIwZERRM0ZIQ2xOTk5EbENRVTFFUVRKclFVMUhXVU5OVVVRNFRtRnpRMGhwVUVOTVFYUnZiRlpQTlM4d2NtdzFWV1pDZVcxMmFubDNhVEJPVmt0QlNtUmpRWFpRTmpFS0wwbEtiRkppV0RWSldtSmhkRWtyT0ZsUGMwTk5VVU5IYUZoT01rSlVOa05VZFdGWFF6TnhkMHhHZWtGRGVWUldOSFZwYmxGVmJraFBla1ZFWXpjMGVnb3hLM1ZIVkcxSVpsWlVVa1JKYVRJdmRFWlJkWFZPVFQwS0xTMHRMUzFGVGtRZ1EwVlNWRWxHU1VOQlZFVXRMUzB0TFFvPSJ9fX19"}], "timestampVerificationData":{"rfc3161Timestamps":[{"signedTimestamp":"MIICyjADAgEAMIICwQYJKoZIhvcNAQcCoIICsjCCAq4CAQMxDTALBglghkgBZQMEAgEwgbcGCyqGSIb3DQEJEAEEoIGnBIGkMIGhAgEBBgkrBgEEAYO/MAIwMTANBglghkgBZQMEAgEFAAQgawL8w2Yhv+alAQlEIxeoeY+ZtrXWuMVUorIiTtT4TyICFGiL4yQnFEcyiQjCaUOyUI1utRZ8GA8yMDI2MTAxOTIyMzExOFowAwIBAaAypDAwLjEVMBMGA1UEChMMc2lnc3RvcmUuZGV2MRUwEwYDVQQDEwxzaWdzdG9yZS10c2GgADGCAdwwggHYAgEBMFEwOTEVMBMGA1UEChMMc2lnc3RvcmUuZGV2MSAwHgYDVQQDExdzaWdzdG9yZS10c2Etc2VsZnNpZ25lZAIUOhNULwyQYe68wUMvy4qOiyojiwwwCwYJYIZIAWUDBAIBoIH8MBoGCSqGSIb3DQEJAzENBgsqhkiG9w0BCRABBDAcBgkqhkiG9w0BCQUxDxcNMjYxMDE5MjIzMTE4WjAvBgkqhkiG9w0BCQQxIgQgKQVaP34iASGj9tDKvbngVn5AU9ONoeeIIpUzaQv1CrQwgY4GCyqGSIb3DQEJEAIvMX8wfTB7MHkEIIX5J7wHq2LKw7RDVsEO/IGyxog/2nq55thw2dE6zQW3MFUwPaQ7MDkxFTATBgNVBAoTDHNpZ3N0b3JlLmRldjEgMB4GA1UEAxMXc2lnc3RvcmUtdHNhLXNlbGZzaWduZWQCFDoTVC8MkGHuvMFDL8uKjosqI4sMMAoGCCqGSM49BAMCBGgwZgIxAOYdsdmqMP1jGpCvbUMDAMn9a7y2Yi0TW0qE804eLILU3GezzrxsPYzwIMjo22GsAAIxAKw3FmgrW1NuQOEFdtHwvQwje7S8o7tQedDvPWScBIyBcVGsh47i6D9JjfP8hsQAyA=="}]}}, "messageSignature":{"messageDigest":{"algorithm":"SHA2_256", "digest":"AIupKj7MkgnQsFGZ7aPLG6TsiMPl+LSvrAD7IZiS20w="}, "signature":"MEQCH1pWo7lERDmxEfY7zfg2Plr2Soljeduk6BdhjaMljfACIQC1ZfK8cuETDhunkODt187M2pvTi8EVs84XL/467DkWmQ=="}}
build/torch213-rocm71-x86_64-linux/mha.py ADDED
@@ -0,0 +1,332 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FlashAttention forward entry points backed by aiter's Composable-Kernel FMHA.
2
+
3
+ This is the compiled (HIP / Composable Kernel) counterpart to the Triton
4
+ ``aiter-flash-attn`` package. Only the *forward* pass is provided (dense and
5
+ variable-length), including support for learnable attention sinks via
6
+ ``window_size[2]`` / ``s_aux`` (e.g. gpt-oss). The op set ships exclusively
7
+ the CK ``mha_fwd`` / ``mha_varlen_fwd`` kernels, so the sink path is always
8
+ taken (no fall-through to a non-sink ASM kernel).
9
+ """
10
+
11
+ from typing import Optional, Tuple, Union
12
+
13
+ import torch
14
+ from torch import Tensor
15
+
16
+ from ._ops import ops
17
+
18
+
19
+ def _round_multiple(x: int, m: int) -> int:
20
+ return (x + m - 1) // m * m
21
+
22
+
23
+ def _maybe_pad_headdim(t: Tensor, target: int) -> Tensor:
24
+ og = t.size(-1)
25
+ if og == target:
26
+ return t
27
+ return torch.nn.functional.pad(t, [0, target - og])
28
+
29
+
30
+ def _unpack_window(window_size) -> Tuple[int, int, int]:
31
+ left = int(window_size[0])
32
+ right = int(window_size[1])
33
+ sink = int(window_size[2]) if len(window_size) == 3 else 0
34
+ return left, right, sink
35
+
36
+
37
+ def _prep_sink(s_aux: Optional[Tensor], nheads: int, device) -> Optional[Tensor]:
38
+ if s_aux is None:
39
+ return None
40
+ assert s_aux.device == device, "s_aux must be on the same device as q"
41
+ assert s_aux.shape[0] == nheads, "s_aux must have shape [nheads_q]"
42
+ if s_aux.dtype != torch.float32:
43
+ s_aux = s_aux.to(torch.float32)
44
+ return s_aux
45
+
46
+
47
+ def _effective_sink_size(sink_size: int, s_aux: Optional[Tensor]) -> int:
48
+ # The CK kernel only takes the sink path when sink_size > 0. Callers using
49
+ # the transformers attention interface pass the per-head sink logits as
50
+ # ``s_aux`` but leave ``window_size`` a 2-tuple (sink_size == 0), so enable
51
+ # a single learnable sink automatically when s_aux is given.
52
+ if s_aux is not None and sink_size == 0:
53
+ return 1
54
+ return sink_size
55
+
56
+
57
+ def flash_attn_func(
58
+ q: Tensor,
59
+ k: Tensor,
60
+ v: Tensor,
61
+ dropout_p: float = 0.0,
62
+ softmax_scale: Optional[float] = None,
63
+ causal: bool = False,
64
+ window_size: Tuple[int, ...] = (-1, -1, 0),
65
+ bias: Optional[Tensor] = None,
66
+ alibi_slopes: Optional[Tensor] = None,
67
+ return_lse: bool = False,
68
+ return_attn_probs: bool = False,
69
+ s_aux: Optional[Tensor] = None,
70
+ **kwargs,
71
+ ) -> Union[Tensor, Tuple[Tensor, ...]]:
72
+ """Dense FlashAttention forward.
73
+
74
+ ``q, k, v`` have shape ``(batch, seqlen, nheads, headdim)``. ``window_size``
75
+ is ``(left, right, sink_size)``; ``s_aux`` is an optional ``(nheads_q,)``
76
+ fp32 tensor of learnable sink logits. Extra keyword arguments (e.g. those
77
+ passed by the transformers flash-attention interface) are ignored.
78
+ """
79
+ if softmax_scale is None:
80
+ softmax_scale = q.shape[-1] ** (-0.5)
81
+ window_size_left, window_size_right, sink_size = _unpack_window(window_size)
82
+
83
+ nheads_q = q.size(2)
84
+ s_aux = _prep_sink(s_aux, nheads_q, q.device)
85
+ sink_size = _effective_sink_size(sink_size, s_aux)
86
+
87
+ head_q_og = q.size(-1)
88
+ head_v_og = v.size(-1)
89
+ head_q = _round_multiple(head_q_og, 8)
90
+ head_v = _round_multiple(head_v_og, 8)
91
+ q = _maybe_pad_headdim(q, head_q)
92
+ k = _maybe_pad_headdim(k, head_q)
93
+ v = _maybe_pad_headdim(v, head_v)
94
+
95
+ return_softmax = return_attn_probs and dropout_p > 0
96
+ out, softmax_lse, S_dmask, _ = ops.mha_fwd(
97
+ q,
98
+ k,
99
+ v,
100
+ dropout_p,
101
+ softmax_scale,
102
+ causal,
103
+ window_size_left,
104
+ window_size_right,
105
+ sink_size,
106
+ return_lse or return_attn_probs,
107
+ return_softmax,
108
+ None, # cu_seqlens_q
109
+ None, # cu_seqlens_kv
110
+ None, # out
111
+ bias,
112
+ alibi_slopes,
113
+ None, # q_descale
114
+ None, # k_descale
115
+ None, # v_descale
116
+ s_aux,
117
+ None, # gen
118
+ )
119
+
120
+ out = out[..., :head_v_og]
121
+ if not (return_lse or return_attn_probs):
122
+ return out
123
+ result = (out,)
124
+ if return_lse:
125
+ result = result + (softmax_lse,)
126
+ if return_attn_probs:
127
+ result = result + (S_dmask,)
128
+ return result
129
+
130
+
131
+ def flash_attn_varlen_func(
132
+ q: Tensor,
133
+ k: Tensor,
134
+ v: Tensor,
135
+ cu_seqlens_q: Tensor,
136
+ cu_seqlens_k: Tensor,
137
+ max_seqlen_q: int,
138
+ max_seqlen_k: int,
139
+ dropout_p: float = 0.0,
140
+ softmax_scale: Optional[float] = None,
141
+ causal: bool = False,
142
+ window_size: Tuple[int, ...] = (-1, -1, 0),
143
+ bias: Optional[Tensor] = None,
144
+ alibi_slopes: Optional[Tensor] = None,
145
+ return_lse: bool = False,
146
+ return_attn_probs: bool = False,
147
+ block_table: Optional[Tensor] = None,
148
+ min_seqlen_q: int = 0,
149
+ logits_soft_cap: float = 0.0,
150
+ zero_tensors: bool = False,
151
+ cu_seqlens_q_padded: Optional[Tensor] = None,
152
+ cu_seqlens_k_padded: Optional[Tensor] = None,
153
+ s_aux: Optional[Tensor] = None,
154
+ **kwargs,
155
+ ) -> Union[Tensor, Tuple[Tensor, ...]]:
156
+ """Variable-length / packed FlashAttention forward.
157
+
158
+ ``q`` has shape ``(total_q, nheads, headdim)``; ``cu_seqlens_*`` are int32
159
+ cumulative offsets of shape ``(batch + 1,)``. Sinks are supported through
160
+ ``window_size[2]`` / ``s_aux`` as in :func:`flash_attn_func`. Extra keyword
161
+ arguments (e.g. from the transformers flash-attention interface) are ignored.
162
+ """
163
+ if softmax_scale is None:
164
+ softmax_scale = q.shape[-1] ** (-0.5)
165
+ window_size_left, window_size_right, sink_size = _unpack_window(window_size)
166
+
167
+ nheads_q = q.size(1)
168
+ s_aux = _prep_sink(s_aux, nheads_q, q.device)
169
+ sink_size = _effective_sink_size(sink_size, s_aux)
170
+
171
+ head_q_og = q.size(-1)
172
+ head_v_og = v.size(-1)
173
+ head_q = _round_multiple(head_q_og, 8)
174
+ head_v = _round_multiple(head_v_og, 8)
175
+ q = _maybe_pad_headdim(q, head_q)
176
+ k = _maybe_pad_headdim(k, head_q)
177
+ v = _maybe_pad_headdim(v, head_v)
178
+
179
+ return_softmax = return_attn_probs and dropout_p > 0
180
+ out, softmax_lse, S_dmask, _ = ops.mha_varlen_fwd(
181
+ q,
182
+ k,
183
+ v,
184
+ cu_seqlens_q,
185
+ cu_seqlens_k,
186
+ max_seqlen_q,
187
+ max_seqlen_k,
188
+ min_seqlen_q,
189
+ dropout_p,
190
+ softmax_scale,
191
+ logits_soft_cap,
192
+ zero_tensors,
193
+ causal,
194
+ window_size_left,
195
+ window_size_right,
196
+ sink_size,
197
+ return_lse or return_attn_probs,
198
+ return_softmax,
199
+ None, # out
200
+ block_table,
201
+ bias,
202
+ alibi_slopes,
203
+ None, # q_descale
204
+ None, # k_descale
205
+ None, # v_descale
206
+ None, # gen
207
+ cu_seqlens_q_padded,
208
+ cu_seqlens_k_padded,
209
+ s_aux,
210
+ )
211
+
212
+ out = out[..., :head_v_og]
213
+ if not (return_lse or return_attn_probs):
214
+ return out
215
+ result = (out,)
216
+ if return_lse:
217
+ result = result + (softmax_lse,)
218
+ if return_attn_probs:
219
+ result = result + (S_dmask,)
220
+ return result
221
+
222
+
223
+ def _append_to_contiguous_cache(k_cache, v_cache, k, v, cache_seqlens, Sq):
224
+ """In-place append of new k/v into a contiguous cache at per-batch offsets."""
225
+ B = k_cache.shape[0]
226
+ if isinstance(cache_seqlens, int):
227
+ cache_seqlens = torch.full((B,), cache_seqlens, device=k_cache.device, dtype=torch.long)
228
+ pos = cache_seqlens.view(B, 1).to(torch.long) + torch.arange(Sq, device=k_cache.device).view(1, Sq)
229
+ bidx = torch.arange(B, device=k_cache.device).view(B, 1).expand(B, Sq)
230
+ k_cache[bidx, pos] = k.to(k_cache.dtype)
231
+ v_cache[bidx, pos] = v.to(v_cache.dtype)
232
+
233
+
234
+ def flash_attn_with_kvcache(
235
+ q: Tensor,
236
+ k_cache: Tensor,
237
+ v_cache: Tensor,
238
+ k: Optional[Tensor] = None,
239
+ v: Optional[Tensor] = None,
240
+ rotary_cos: Optional[Tensor] = None,
241
+ rotary_sin: Optional[Tensor] = None,
242
+ cache_seqlens=None,
243
+ block_table: Optional[Tensor] = None,
244
+ softmax_scale: Optional[float] = None,
245
+ causal: bool = True,
246
+ window_size: Tuple[int, ...] = (-1, -1, 0),
247
+ s_aux: Optional[Tensor] = None,
248
+ return_lse: bool = False,
249
+ **kwargs,
250
+ ) -> Union[Tensor, Tuple[Tensor, ...]]:
251
+ """FlashAttention against a KV cache, built on the CK split-KV kernel.
252
+
253
+ ``q`` is ``(batch, seqlen_q, nheads, headdim)``. For a contiguous cache,
254
+ ``k_cache``/``v_cache`` are ``(batch, seqlen_cache, nheads_k, headdim)`` and
255
+ ``cache_seqlens`` (int or ``(batch,)``) is the number of valid cache entries
256
+ *before* appending. If ``k``/``v`` (the new tokens) are given, they are
257
+ written into the cache in place. The query attends to the valid prefix
258
+ (``cache_seqlens`` + ``seqlen_q``) with bottom-right causal alignment.
259
+
260
+ Notes / current limitations:
261
+ - Rotary embedding is not applied here; pass ``q``/``k`` already rotated
262
+ (``rotary_cos``/``rotary_sin`` are rejected).
263
+ - Paged caches are addressed via ``block_table`` (forwarded to the split-KV
264
+ varlen kernel); in-cache append for paged layouts must be done by the
265
+ caller (pass ``k=v=None``).
266
+ """
267
+ if rotary_cos is not None or rotary_sin is not None:
268
+ raise NotImplementedError(
269
+ "flash_attn_with_kvcache does not apply rotary; rotate q/k beforehand."
270
+ )
271
+ if softmax_scale is None:
272
+ softmax_scale = q.shape[-1] ** (-0.5)
273
+ window_size_left, window_size_right, sink_size = _unpack_window(window_size)
274
+ B, Sq, Hq, D = q.shape
275
+ s_aux = _prep_sink(s_aux, Hq, q.device)
276
+ sink_size = _effective_sink_size(sink_size, s_aux)
277
+
278
+ paged = block_table is not None
279
+
280
+ # ---- append new tokens into the cache (contiguous layout only) ----
281
+ if k is not None and not paged:
282
+ assert cache_seqlens is not None, "cache_seqlens required to append into the cache"
283
+ _append_to_contiguous_cache(k_cache, v_cache, k, v, cache_seqlens, Sq)
284
+ elif k is not None and paged:
285
+ raise NotImplementedError(
286
+ "in-cache append for paged layouts is not supported; append externally "
287
+ "and call with k=v=None."
288
+ )
289
+
290
+ appended = 0 if k is None else Sq
291
+
292
+ # ---- key lengths ----
293
+ if cache_seqlens is None:
294
+ klens = torch.full((B,), k_cache.shape[1], device=q.device, dtype=torch.long)
295
+ elif isinstance(cache_seqlens, int):
296
+ klens = torch.full((B,), cache_seqlens + appended, device=q.device, dtype=torch.long)
297
+ else:
298
+ klens = cache_seqlens.to(q.device, torch.long) + appended
299
+
300
+ uniform = bool((klens == klens[0]).all().item())
301
+
302
+ # ---- fast path: contiguous cache, uniform length -> dense kernel ----
303
+ if not paged and uniform:
304
+ L = int(klens[0].item())
305
+ return flash_attn_func(
306
+ q, k_cache[:, :L], v_cache[:, :L],
307
+ softmax_scale=softmax_scale, causal=causal,
308
+ window_size=(window_size_left, window_size_right, sink_size),
309
+ s_aux=s_aux, return_lse=return_lse,
310
+ )
311
+
312
+ # ---- general path: variable cache lengths -> split-KV varlen kernel ----
313
+ cu_q = torch.arange(0, (B + 1) * Sq, Sq, device=q.device, dtype=torch.int32)
314
+ cu_k = torch.zeros(B + 1, device=q.device, dtype=torch.int32)
315
+ cu_k[1:] = torch.cumsum(klens, 0).to(torch.int32)
316
+ q_packed = q.reshape(B * Sq, Hq, D)
317
+ if paged:
318
+ k_packed, v_packed = k_cache, v_cache # kernel gathers via block_table
319
+ else:
320
+ k_packed = torch.cat([k_cache[b, : int(klens[b])] for b in range(B)], dim=0)
321
+ v_packed = torch.cat([v_cache[b, : int(klens[b])] for b in range(B)], dim=0)
322
+
323
+ out = flash_attn_varlen_func(
324
+ q_packed, k_packed, v_packed, cu_q, cu_k, Sq, int(klens.max().item()),
325
+ softmax_scale=softmax_scale, causal=causal,
326
+ window_size=(window_size_left, window_size_right, sink_size),
327
+ block_table=block_table, s_aux=s_aux, return_lse=return_lse,
328
+ )
329
+ if return_lse:
330
+ out, lse = out
331
+ return out.reshape(B, Sq, Hq, D), lse
332
+ return out.reshape(B, Sq, Hq, D)
build/torch213-rocm71-x86_64-linux/symbols.json ADDED
@@ -0,0 +1,349 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "schema_version": 1,
3
+ "module": "_aiter_flash_attn_ck_rocm_fe2bb66",
4
+ "functions": [
5
+ {
6
+ "name": "flash_attn_func",
7
+ "qualname": "flash_attn_func",
8
+ "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha",
9
+ "kind": "function",
10
+ "doc": "Dense FlashAttention forward.\n\n``q, k, v`` have shape ``(batch, seqlen, nheads, headdim)``. ``window_size``\nis ``(left, right, sink_size)``; ``s_aux`` is an optional ``(nheads_q,)``\nfp32 tensor of learnable sink logits. Extra keyword arguments (e.g. those\npassed by the transformers flash-attention interface) are ignored.",
11
+ "signature": {
12
+ "parameters": [
13
+ {
14
+ "name": "q",
15
+ "kind": "POSITIONAL_OR_KEYWORD",
16
+ "default": null,
17
+ "annotation": "torch.Tensor"
18
+ },
19
+ {
20
+ "name": "k",
21
+ "kind": "POSITIONAL_OR_KEYWORD",
22
+ "default": null,
23
+ "annotation": "torch.Tensor"
24
+ },
25
+ {
26
+ "name": "v",
27
+ "kind": "POSITIONAL_OR_KEYWORD",
28
+ "default": null,
29
+ "annotation": "torch.Tensor"
30
+ },
31
+ {
32
+ "name": "dropout_p",
33
+ "kind": "POSITIONAL_OR_KEYWORD",
34
+ "default": "0.0",
35
+ "annotation": "float"
36
+ },
37
+ {
38
+ "name": "softmax_scale",
39
+ "kind": "POSITIONAL_OR_KEYWORD",
40
+ "default": "None",
41
+ "annotation": "float | None"
42
+ },
43
+ {
44
+ "name": "causal",
45
+ "kind": "POSITIONAL_OR_KEYWORD",
46
+ "default": "False",
47
+ "annotation": "bool"
48
+ },
49
+ {
50
+ "name": "window_size",
51
+ "kind": "POSITIONAL_OR_KEYWORD",
52
+ "default": "(-1, -1, 0)",
53
+ "annotation": "Tuple[int, ...]"
54
+ },
55
+ {
56
+ "name": "bias",
57
+ "kind": "POSITIONAL_OR_KEYWORD",
58
+ "default": "None",
59
+ "annotation": "torch.Tensor | None"
60
+ },
61
+ {
62
+ "name": "alibi_slopes",
63
+ "kind": "POSITIONAL_OR_KEYWORD",
64
+ "default": "None",
65
+ "annotation": "torch.Tensor | None"
66
+ },
67
+ {
68
+ "name": "return_lse",
69
+ "kind": "POSITIONAL_OR_KEYWORD",
70
+ "default": "False",
71
+ "annotation": "bool"
72
+ },
73
+ {
74
+ "name": "return_attn_probs",
75
+ "kind": "POSITIONAL_OR_KEYWORD",
76
+ "default": "False",
77
+ "annotation": "bool"
78
+ },
79
+ {
80
+ "name": "s_aux",
81
+ "kind": "POSITIONAL_OR_KEYWORD",
82
+ "default": "None",
83
+ "annotation": "torch.Tensor | None"
84
+ },
85
+ {
86
+ "name": "kwargs",
87
+ "kind": "VAR_KEYWORD",
88
+ "default": null,
89
+ "annotation": null
90
+ }
91
+ ],
92
+ "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]"
93
+ }
94
+ },
95
+ {
96
+ "name": "flash_attn_varlen_func",
97
+ "qualname": "flash_attn_varlen_func",
98
+ "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha",
99
+ "kind": "function",
100
+ "doc": "Variable-length / packed FlashAttention forward.\n\n``q`` has shape ``(total_q, nheads, headdim)``; ``cu_seqlens_*`` are int32\ncumulative offsets of shape ``(batch + 1,)``. Sinks are supported through\n``window_size[2]`` / ``s_aux`` as in :func:`flash_attn_func`. Extra keyword\narguments (e.g. from the transformers flash-attention interface) are ignored.",
101
+ "signature": {
102
+ "parameters": [
103
+ {
104
+ "name": "q",
105
+ "kind": "POSITIONAL_OR_KEYWORD",
106
+ "default": null,
107
+ "annotation": "torch.Tensor"
108
+ },
109
+ {
110
+ "name": "k",
111
+ "kind": "POSITIONAL_OR_KEYWORD",
112
+ "default": null,
113
+ "annotation": "torch.Tensor"
114
+ },
115
+ {
116
+ "name": "v",
117
+ "kind": "POSITIONAL_OR_KEYWORD",
118
+ "default": null,
119
+ "annotation": "torch.Tensor"
120
+ },
121
+ {
122
+ "name": "cu_seqlens_q",
123
+ "kind": "POSITIONAL_OR_KEYWORD",
124
+ "default": null,
125
+ "annotation": "torch.Tensor"
126
+ },
127
+ {
128
+ "name": "cu_seqlens_k",
129
+ "kind": "POSITIONAL_OR_KEYWORD",
130
+ "default": null,
131
+ "annotation": "torch.Tensor"
132
+ },
133
+ {
134
+ "name": "max_seqlen_q",
135
+ "kind": "POSITIONAL_OR_KEYWORD",
136
+ "default": null,
137
+ "annotation": "int"
138
+ },
139
+ {
140
+ "name": "max_seqlen_k",
141
+ "kind": "POSITIONAL_OR_KEYWORD",
142
+ "default": null,
143
+ "annotation": "int"
144
+ },
145
+ {
146
+ "name": "dropout_p",
147
+ "kind": "POSITIONAL_OR_KEYWORD",
148
+ "default": "0.0",
149
+ "annotation": "float"
150
+ },
151
+ {
152
+ "name": "softmax_scale",
153
+ "kind": "POSITIONAL_OR_KEYWORD",
154
+ "default": "None",
155
+ "annotation": "float | None"
156
+ },
157
+ {
158
+ "name": "causal",
159
+ "kind": "POSITIONAL_OR_KEYWORD",
160
+ "default": "False",
161
+ "annotation": "bool"
162
+ },
163
+ {
164
+ "name": "window_size",
165
+ "kind": "POSITIONAL_OR_KEYWORD",
166
+ "default": "(-1, -1, 0)",
167
+ "annotation": "Tuple[int, ...]"
168
+ },
169
+ {
170
+ "name": "bias",
171
+ "kind": "POSITIONAL_OR_KEYWORD",
172
+ "default": "None",
173
+ "annotation": "torch.Tensor | None"
174
+ },
175
+ {
176
+ "name": "alibi_slopes",
177
+ "kind": "POSITIONAL_OR_KEYWORD",
178
+ "default": "None",
179
+ "annotation": "torch.Tensor | None"
180
+ },
181
+ {
182
+ "name": "return_lse",
183
+ "kind": "POSITIONAL_OR_KEYWORD",
184
+ "default": "False",
185
+ "annotation": "bool"
186
+ },
187
+ {
188
+ "name": "return_attn_probs",
189
+ "kind": "POSITIONAL_OR_KEYWORD",
190
+ "default": "False",
191
+ "annotation": "bool"
192
+ },
193
+ {
194
+ "name": "block_table",
195
+ "kind": "POSITIONAL_OR_KEYWORD",
196
+ "default": "None",
197
+ "annotation": "torch.Tensor | None"
198
+ },
199
+ {
200
+ "name": "min_seqlen_q",
201
+ "kind": "POSITIONAL_OR_KEYWORD",
202
+ "default": "0",
203
+ "annotation": "int"
204
+ },
205
+ {
206
+ "name": "logits_soft_cap",
207
+ "kind": "POSITIONAL_OR_KEYWORD",
208
+ "default": "0.0",
209
+ "annotation": "float"
210
+ },
211
+ {
212
+ "name": "zero_tensors",
213
+ "kind": "POSITIONAL_OR_KEYWORD",
214
+ "default": "False",
215
+ "annotation": "bool"
216
+ },
217
+ {
218
+ "name": "cu_seqlens_q_padded",
219
+ "kind": "POSITIONAL_OR_KEYWORD",
220
+ "default": "None",
221
+ "annotation": "torch.Tensor | None"
222
+ },
223
+ {
224
+ "name": "cu_seqlens_k_padded",
225
+ "kind": "POSITIONAL_OR_KEYWORD",
226
+ "default": "None",
227
+ "annotation": "torch.Tensor | None"
228
+ },
229
+ {
230
+ "name": "s_aux",
231
+ "kind": "POSITIONAL_OR_KEYWORD",
232
+ "default": "None",
233
+ "annotation": "torch.Tensor | None"
234
+ },
235
+ {
236
+ "name": "kwargs",
237
+ "kind": "VAR_KEYWORD",
238
+ "default": null,
239
+ "annotation": null
240
+ }
241
+ ],
242
+ "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]"
243
+ }
244
+ },
245
+ {
246
+ "name": "flash_attn_with_kvcache",
247
+ "qualname": "flash_attn_with_kvcache",
248
+ "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha",
249
+ "kind": "function",
250
+ "doc": "FlashAttention against a KV cache, built on the CK split-KV kernel.\n\n``q`` is ``(batch, seqlen_q, nheads, headdim)``. For a contiguous cache,\n``k_cache``/``v_cache`` are ``(batch, seqlen_cache, nheads_k, headdim)`` and\n``cache_seqlens`` (int or ``(batch,)``) is the number of valid cache entries\n*before* appending. If ``k``/``v`` (the new tokens) are given, they are\nwritten into the cache in place. The query attends to the valid prefix\n(``cache_seqlens`` + ``seqlen_q``) with bottom-right causal alignment.\n\nNotes / current limitations:\n- Rotary embedding is not applied here; pass ``q``/``k`` already rotated\n (``rotary_cos``/``rotary_sin`` are rejected).\n- Paged caches are addressed via ``block_table`` (forwarded to the split-KV\n varlen kernel); in-cache append for paged layouts must be done by the\n caller (pass ``k=v=None``).",
251
+ "signature": {
252
+ "parameters": [
253
+ {
254
+ "name": "q",
255
+ "kind": "POSITIONAL_OR_KEYWORD",
256
+ "default": null,
257
+ "annotation": "torch.Tensor"
258
+ },
259
+ {
260
+ "name": "k_cache",
261
+ "kind": "POSITIONAL_OR_KEYWORD",
262
+ "default": null,
263
+ "annotation": "torch.Tensor"
264
+ },
265
+ {
266
+ "name": "v_cache",
267
+ "kind": "POSITIONAL_OR_KEYWORD",
268
+ "default": null,
269
+ "annotation": "torch.Tensor"
270
+ },
271
+ {
272
+ "name": "k",
273
+ "kind": "POSITIONAL_OR_KEYWORD",
274
+ "default": "None",
275
+ "annotation": "torch.Tensor | None"
276
+ },
277
+ {
278
+ "name": "v",
279
+ "kind": "POSITIONAL_OR_KEYWORD",
280
+ "default": "None",
281
+ "annotation": "torch.Tensor | None"
282
+ },
283
+ {
284
+ "name": "rotary_cos",
285
+ "kind": "POSITIONAL_OR_KEYWORD",
286
+ "default": "None",
287
+ "annotation": "torch.Tensor | None"
288
+ },
289
+ {
290
+ "name": "rotary_sin",
291
+ "kind": "POSITIONAL_OR_KEYWORD",
292
+ "default": "None",
293
+ "annotation": "torch.Tensor | None"
294
+ },
295
+ {
296
+ "name": "cache_seqlens",
297
+ "kind": "POSITIONAL_OR_KEYWORD",
298
+ "default": "None",
299
+ "annotation": null
300
+ },
301
+ {
302
+ "name": "block_table",
303
+ "kind": "POSITIONAL_OR_KEYWORD",
304
+ "default": "None",
305
+ "annotation": "torch.Tensor | None"
306
+ },
307
+ {
308
+ "name": "softmax_scale",
309
+ "kind": "POSITIONAL_OR_KEYWORD",
310
+ "default": "None",
311
+ "annotation": "float | None"
312
+ },
313
+ {
314
+ "name": "causal",
315
+ "kind": "POSITIONAL_OR_KEYWORD",
316
+ "default": "True",
317
+ "annotation": "bool"
318
+ },
319
+ {
320
+ "name": "window_size",
321
+ "kind": "POSITIONAL_OR_KEYWORD",
322
+ "default": "(-1, -1, 0)",
323
+ "annotation": "Tuple[int, ...]"
324
+ },
325
+ {
326
+ "name": "s_aux",
327
+ "kind": "POSITIONAL_OR_KEYWORD",
328
+ "default": "None",
329
+ "annotation": "torch.Tensor | None"
330
+ },
331
+ {
332
+ "name": "return_lse",
333
+ "kind": "POSITIONAL_OR_KEYWORD",
334
+ "default": "False",
335
+ "annotation": "bool"
336
+ },
337
+ {
338
+ "name": "kwargs",
339
+ "kind": "VAR_KEYWORD",
340
+ "default": null,
341
+ "annotation": null
342
+ }
343
+ ],
344
+ "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]"
345
+ }
346
+ }
347
+ ],
348
+ "layers": []
349
+ }
build/torch213-rocm72-x86_64-linux/__init__.py ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """AITER Composable-Kernel FlashAttention kernels for AMD ROCm (compiled HIP).
2
+
3
+ Compiled (Composable Kernel / HIP) counterpart to the Triton-based
4
+ ``aiter-flash-attn`` package, repackaged from the CK FMHA forward path of the
5
+ `ROCm/aiter <https://github.com/ROCm/aiter>`_ project as a self-contained
6
+ Hugging Face Hub kernel. Exposes the FlashAttention forward entry points with
7
+ learnable attention sink support (e.g. gpt-oss).
8
+ """
9
+
10
+ from .mha import (
11
+ flash_attn_func,
12
+ flash_attn_varlen_func,
13
+ flash_attn_with_kvcache,
14
+ )
15
+
16
+
17
+ __kernel_metadata__ = {
18
+ "license": "mit",
19
+ }
20
+
21
+
22
+ __all__ = [
23
+ "__kernel_metadata__",
24
+ "flash_attn_func",
25
+ "flash_attn_varlen_func",
26
+ "flash_attn_with_kvcache",
27
+ ]