diff --git a/build/torch210-rocm70-x86_64-linux/__init__.py b/build/torch210-rocm70-x86_64-linux/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..73acb58c1142a1ab2a6ed9d6f34bc6a53ab5e9b5 --- /dev/null +++ b/build/torch210-rocm70-x86_64-linux/__init__.py @@ -0,0 +1,27 @@ +"""AITER Composable-Kernel FlashAttention kernels for AMD ROCm (compiled HIP). + +Compiled (Composable Kernel / HIP) counterpart to the Triton-based +``aiter-flash-attn`` package, repackaged from the CK FMHA forward path of the +`ROCm/aiter `_ project as a self-contained +Hugging Face Hub kernel. Exposes the FlashAttention forward entry points with +learnable attention sink support (e.g. gpt-oss). +""" + +from .mha import ( + flash_attn_func, + flash_attn_varlen_func, + flash_attn_with_kvcache, +) + + +__kernel_metadata__ = { + "license": "mit", +} + + +__all__ = [ + "__kernel_metadata__", + "flash_attn_func", + "flash_attn_varlen_func", + "flash_attn_with_kvcache", +] diff --git a/build/torch210-rocm70-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so b/build/torch210-rocm70-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so new file mode 100644 index 0000000000000000000000000000000000000000..ba8c6bd7e5dc32d0bf69498b473ddc92591ccc6c --- /dev/null +++ b/build/torch210-rocm70-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8edc92287fd49261db7c4799b29691a77e7db509ec44dac8bf85fba127f0e641 +size 204329576 diff --git a/build/torch210-rocm70-x86_64-linux/_ops.py b/build/torch210-rocm70-x86_64-linux/_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..2b4dd56c220b61fd2c16e0c1a02ee91a491cef0c --- /dev/null +++ b/build/torch210-rocm70-x86_64-linux/_ops.py @@ -0,0 +1,9 @@ +import torch +from . import _aiter_flash_attn_ck_rocm_fe2bb66 +ops = torch.ops._aiter_flash_attn_ck_rocm_fe2bb66 + +def add_op_namespace_prefix(op_name: str): + """ + Prefix op by namespace. + """ + return f"_aiter_flash_attn_ck_rocm_fe2bb66::{op_name}" diff --git a/build/torch210-rocm70-x86_64-linux/metadata.json b/build/torch210-rocm70-x86_64-linux/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..8c3f109e12797885540f18a6e52b6a6e97026e2c --- /dev/null +++ b/build/torch210-rocm70-x86_64-linux/metadata.json @@ -0,0 +1,37 @@ +{ + "name": "aiter-flash-attn-ck", + "id": "_aiter_flash_attn_ck_rocm_fe2bb66", + "version": 1, + "kernels-minver": "0.14.0", + "license": "MIT", + "python-depends": [], + "kernel-depends": [], + "backend": { + "type": "rocm", + "archs": [ + "gfx942", + "gfx950" + ] + }, + "digest": { + "algorithm": "sha256", + "files": { + "__init__.py": "7wj1GTm2bqTLLCyT4xLuAk/+JWYrhINjY/ODMNWhgZ0=", + "_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so": "jtySKH/UkmHbfEeZspaRp359tQnsRNrIv4X7oSfw5kE=", + "_ops.py": "4WkdYDWWJp60zep+oW/5ym9vIeVMNfrgRCkP3o/w7yU=", + "mha.py": "zCAjY4O+DiCzmEL+W9sqXxZaqjsj4RJbWFcyPpUZejk=", + "symbols.json": "rhnF4H8Jsb6I+YPxIVFMgnLcTQJODEXbZ7ge77NZXAU=" + } + }, + "provenance": { + "kernel-builder": { + "version": "0.18.0-dev0", + "commit": "d054906a13710b811b96697b4635f03020842733", + "dirty": false + }, + "kernel": { + "commit": "fe2bb66fee6c2a77eccf08a0508dc57428d488ed", + "dirty": false + } + } +} \ No newline at end of file diff --git a/build/torch210-rocm70-x86_64-linux/metadata.json.sigstore b/build/torch210-rocm70-x86_64-linux/metadata.json.sigstore new file mode 100644 index 0000000000000000000000000000000000000000..ed1c8de428485a4fd624ed0970937019c3f84ff4 --- /dev/null +++ b/build/torch210-rocm70-x86_64-linux/metadata.json.sigstore @@ -0,0 +1 @@ +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json", "verificationMaterial":{"certificate":{"rawBytes":"MIIHLDCCBrOgAwIBAgIURaYLBvCxo8VXbDXCmafFy5NEz4IwCgYIKoZIzj0EAwMwNzEVMBMGA1UEChMMc2lnc3RvcmUuZGV2MR4wHAYDVQQDExVzaWdzdG9yZS1pbnRlcm1lZGlhdGUwHhcNMjYxMDE5MjIzMTEwWhcNMjYxMDE5MjI0MTEwWjAAMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEj/NhfZLBdSTQaFcS2gIM0IgWG5P9kCv+yNljBxYjIPdh5dPlT5SG3L0TOIhLASBQ2EJ2qqDHsaMED9/WJmH9hqOCBdIwggXOMA4GA1UdDwEB/wQEAwIHgDATBgNVHSUEDDAKBggrBgEFBQcDAzAfBgNVHSMEGDAWgBTf0+nPViQRlvmo2OkoVaLGLhhkPzBrBgNVHREBAf8EYTBfhl1odHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkvLmdpdGh1Yi93b3JrZmxvd3MvYnVpbGQueWFtbEByZWZzL2hlYWRzL21haW4wOQYKKwYBBAGDvzABAQQraHR0cHM6Ly90b2tlbi5hY3Rpb25zLmdpdGh1YnVzZXJjb250ZW50LmNvbTAfBgorBgEEAYO/MAECBBF3b3JrZmxvd19kaXNwYXRjaDA2BgorBgEEAYO/MAEDBChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMBMGCisGAQQBg78wAQQEBUJ1aWxkMCsGCisGAQQBg78wAQUEHWh1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5MB0GCisGAQQBg78wAQYED3JlZnMvaGVhZHMvbWFpbjA7BgorBgEEAYO/MAEIBC0MK2h0dHBzOi8vdG9rZW4uYWN0aW9ucy5naXRodWJ1c2VyY29udGVudC5jb20wbQYKKwYBBAGDvzABCQRfDF1odHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkvLmdpdGh1Yi93b3JrZmxvd3MvYnVpbGQueWFtbEByZWZzL2hlYWRzL21haW4wOAYKKwYBBAGDvzABCgQqDChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMBsGCisGAQQBg78wAQsEDQwLc2VsZi1ob3N0ZWQwQAYKKwYBBAGDvzABDAQyDDBodHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkwOAYKKwYBBAGDvzABDQQqDChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMB8GCisGAQQBg78wAQ4EEQwPcmVmcy9oZWFkcy9tYWluMBoGCisGAQQBg78wAQ8EDAwKMTA3MTQ3NTUyOTAuBgorBgEEAYO/MAEQBCAMHmh0dHBzOi8vZ2l0aHViLmNvbS9odWdnaW5nZmFjZTAYBgorBgEEAYO/MAERBAoMCDI1NzIwNzQzMG0GCisGAQQBg78wARIEXwxdaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5Ly5naXRodWIvd29ya2Zsb3dzL2J1aWxkLnlhbWxAcmVmcy9oZWFkcy9tYWluMDgGCisGAQQBg78wARMEKgwoZmUyYmI2NmZlZTZjMmE3N2VjY2YwOGEwNTA4ZGM1NzQyOGQ0ODhlZDAhBgorBgEEAYO/MAEUBBMMEXdvcmtmbG93X2Rpc3BhdGNoMGQGCisGAQQBg78wARUEVgxUaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5L2FjdGlvbnMvcnVucy8zOTIzOTUxODEwNC9hdHRlbXB0cy8xMBYGCisGAQQBg78wARYECAwGcHVibGljMEYGCisGAQQBg78wARgEOAw2cmVwbzpodWdnaW5nZmFjZS9rZXJuZWxzLWNvbW11bml0eTpyZWY6cmVmcy9oZWFkcy9tYWluMIGLBgorBgEEAdZ5AgQCBH0EewB5AHcA3T0wasbHETJjGR4cmWc3AqJKXrjePK3/h4pygC8p7o4AAAGhVkpFsgAABAMASDBGAiEAs9cSP0P/pRZ4prabNNIq89TS0RhX2f3JRpHryeIhR3YCIQDnQuuXxSnnnfl8Q7URBS7DC23YrDaSpshSRUENXthSezAKBggqhkjOPQQDAwNnADBkAjEA4gCCe/00DKR/78FksRs11BgMWSSsisk8ME6OMyx2ZOKIvMVy1VHI15E6Xfr3isA+Ai9ZI6Otv6AnOVw/zUHKLvuj6VoYdLp26TwhWYN2N8LFZjzVGNX+0NcN8O6rYcPUsw=="}, "tlogEntries":[{"logIndex":"3327211074", "logId":{"keyId":"wNI9atQGlz+VWfO6LRygH4QUfY/8W4RFwiT5i5WRgB0="}, "kindVersion":{"kind":"hashedrekord", "version":"0.0.1"}, "integratedTime":"1792449070", "inclusionPromise":{"signedEntryTimestamp":"MEUCIBvKLoqT8cbWQvPal/N1/qcIkVmZMOCOjE6UEkoLG/MqAiEAviJ70KCt6sreW85HkoeGGZY7csZL/nOGK/uLJB3Mo5Y="}, "inclusionProof":{"logIndex":"3205306812", "rootHash":"95EwDSNtaCqCTTYN6UQnUbg7LfPk37Dr84DcE4k1RBM=", "treeSize":"3205306841", "hashes":["Ocn28evxoK5pHLcp+On0tDY2wedQVBaFskplXSixJIA=", "mwzUJi5lok6Hybu0CxcVdWew/1Qbhdd1iGjTjEYBcZY=", "5H4knr1LMewHY/JuNzQYKyCx9Q74EBBYz3NqEsl4bQo=", "QZRIvZ2WmlY+tW/q10eYuY+Pr8vI5My3TuMAw/2EObw=", "k+ztsFzU6f6Y66AdU9NwlAXHqLCPJ6LKyl669zO1Hr4=", "ozBdZyx9tTLcaQf2DOlQzhq1u90wECTuyt20Ufxet/w=", "ddB/VAJce0tlDaSu+1n3gD4CW0cuWQCIYTwHV3ehNRg=", "mnhxYPYaPxA2/IDhvMQ+URxfzi+3aVt0cMthNxkCByA=", "jhDPXc61HrM2wJE+UCBVCldN7S5Fn1BuEfb57UT4fLc=", "5egUlZ4Pi0Ix/qLyCDXocsWywJCHqrde/S06vOZvlOM=", "QsXufUpoKtSjKAB19K1G00umQpWvPHlBNuRGFiiVjQQ=", "QIMkwQbwAewhLJd//5FFwI3SnyNQBGjCgJrelfG8vhc=", "Et17CACUv9QN64irUJuBjsrgRGkGnnJ9xacRV/aBdwQ=", "lqgEwkxhDMs/gdIAE4lmjfMwg7nDmv7lda6O0LGliCw=", "HovTPBWzoErvUg1u5sIi3x1ZonZ7yJekVsBrmpomCIA=", "G+nk7mgE5h6dUmv2UY7NDJrY3WvrZY31raaO1AETt+o=", "qlTWNDzERnTBQVcHHXXVPjiUP0SsikGxfcxVlgqVFDo=", "gg9GNApNke3+zNhl4tsWtmIycCFrOMCB3lys6XuSQNQ=", "NlOclldmKPA1tMKIskqHAwuQDYHp31YwpK+OOMrMccc=", "qxzHanAzz57SDdmJe0B7bJK72NTIBbwEMGBBKvDOROw=", "xH/DCseLHr9eKoYT8qsORZK7zVdEGYWHuVtsVrD95wY="], "checkpoint":{"envelope":"rekor.sigstore.dev - 1193050959916656506\n3205306841\n95EwDSNtaCqCTTYN6UQnUbg7LfPk37Dr84DcE4k1RBM=\n\n— rekor.sigstore.dev wNI9ajBFAiEAr3pHRHsXHEeGuhek+3BjAqkphYe6M+vo6+TlCdjCZNYCIE11bmNslf4u1xUYgIQHRJqsJ7XyHOBWS7628PjJh16z\n"}}, "canonicalizedBody":"eyJhcGlWZXJzaW9uIjoiMC4wLjEiLCJraW5kIjoiaGFzaGVkcmVrb3JkIiwic3BlYyI6eyJkYXRhIjp7Imhhc2giOnsiYWxnb3JpdGhtIjoic2hhMjU2IiwidmFsdWUiOiIwOGZhMGNkNTBhODRmOTQxMjc2YzVlNmQ1ZmE0OWYxZDRlMDFlMTU5ODdhNjBhYWI4ZjJhYjZlMzE5ZWVjNzdiIn19LCJzaWduYXR1cmUiOnsiY29udGVudCI6Ik1FVUNJQ1BTaWFYOXBZL29aYnp1YXNHQW5PUTB1TDhQL0pMSEsrRkttVUhZZnZUMkFpRUFzOEZDOEhSeVZFZmo3QUdqYnNJNlZmb2FoY2NLWHM0RkNmY3luZ1RzUVhjPSIsInB1YmxpY0tleSI6eyJjb250ZW50IjoiTFMwdExTMUNSVWRKVGlCRFJWSlVTVVpKUTBGVVJTMHRMUzB0Q2sxSlNVaE1SRU5EUW5KUFowRjNTVUpCWjBsVlVtRlpURUoyUTNodk9GWllZa1JZUTIxaFprWjVOVTVGZWpSSmQwTm5XVWxMYjFwSmVtb3dSVUYzVFhjS1RucEZWazFDVFVkQk1WVkZRMmhOVFdNeWJHNWpNMUoyWTIxVmRWcEhWakpOVWpSM1NFRlpSRlpSVVVSRmVGWjZZVmRrZW1SSE9YbGFVekZ3WW01U2JBcGpiVEZzV2tkc2FHUkhWWGRJYUdOT1RXcFplRTFFUlRWTmFrbDZUVlJGZDFkb1kwNU5hbGw0VFVSRk5VMXFTVEJOVkVWM1YycEJRVTFHYTNkRmQxbElDa3R2V2tsNmFqQkRRVkZaU1V0dldrbDZhakJFUVZGalJGRm5RVVZxTDA1b1pscE1RbVJUVkZGaFJtTlRNbWRKVFRCSloxZEhOVkE1YTBOMkszbE9iR29LUW5oWmFrbFFaR2cxWkZCc1ZEVlRSek5NTUZSUFNXaE1RVk5DVVRKRlNqSnhjVVJJYzJGTlJVUTVMMWRLYlVnNWFIRlBRMEprU1hkbloxaFBUVUUwUndwQk1WVmtSSGRGUWk5M1VVVkJkMGxJWjBSQlZFSm5UbFpJVTFWRlJFUkJTMEpuWjNKQ1owVkdRbEZqUkVGNlFXWkNaMDVXU0ZOTlJVZEVRVmRuUWxSbUNqQXJibEJXYVZGU2JIWnRiekpQYTI5V1lVeEhUR2hvYTFCNlFuSkNaMDVXU0ZKRlFrRm1PRVZaVkVKbWFHd3hiMlJJVW5kamVtOTJUREprY0dSSGFERUtXV2sxYW1JeU1IWmhTRlp1V2pKc2RWb3lXbWhaTWxWMllUSldlV0p0Vm5OamVURnFZakl4ZEdSWE5YQmtTR3QyVEcxa2NHUkhhREZaYVRrellqTktjZ3BhYlhoMlpETk5kbGx1Vm5CaVIxRjFaVmRHZEdKRlFubGFWMXA2VERKb2JGbFhVbnBNTWpGb1lWYzBkMDlSV1V0TGQxbENRa0ZIUkhaNlFVSkJVVkZ5Q21GSVVqQmpTRTAyVEhrNU1HSXlkR3hpYVRWb1dUTlNjR0l5TlhwTWJXUndaRWRvTVZsdVZucGFXRXBxWWpJMU1GcFhOVEJNYlU1MllsUkJaa0puYjNJS1FtZEZSVUZaVHk5TlFVVkRRa0pHTTJJelNuSmFiWGgyWkRFNWEyRllUbmRaV0ZKcVlVUkJNa0puYjNKQ1owVkZRVmxQTDAxQlJVUkNRMmh0V2xSS2FRcFphbGt5V20xV2JFNXRUWGxaVkdNeldsZE9hbHBxUVRSWlZFRXhUVVJvYTFsNlZUTk9SRWswV2tSUk5FOUhWbXROUWsxSFEybHpSMEZSVVVKbk56aDNDa0ZSVVVWQ1ZVb3hZVmQ0YTAxRGMwZERhWE5IUVZGUlFtYzNPSGRCVVZWRlNGZG9NVm95WkhCaWJXUnRXVmRPYkV3eWRHeGpiVFZzWWtoTmRGa3lPWFFLWWxoV2RXRllValZOUWpCSFEybHpSMEZSVVVKbk56aDNRVkZaUlVRelNteGFiazEyWVVkV2FGcElUWFppVjBad1ltcEJOMEpuYjNKQ1owVkZRVmxQTHdwTlFVVkpRa013VFVzeWFEQmtTRUo2VDJrNGRtUkhPWEphVnpSMVdWZE9NR0ZYT1hWamVUVnVZVmhTYjJSWFNqRmpNbFo1V1RJNWRXUkhWblZrUXpWcUNtSXlNSGRpVVZsTFMzZFpRa0pCUjBSMmVrRkNRMUZTWmtSR01XOWtTRkozWTNwdmRrd3laSEJrUjJneFdXazFhbUl5TUhaaFNGWnVXakpzZFZveVdtZ0tXVEpWZG1FeVZubGliVlp6WTNreGFtSXlNWFJrVnpWd1pFaHJka3h0WkhCa1IyZ3hXV2s1TTJJelNuSmFiWGgyWkROTmRsbHVWbkJpUjFGMVpWZEdkQXBpUlVKNVdsZGFla3d5YUd4WlYxSjZUREl4YUdGWE5IZFBRVmxMUzNkWlFrSkJSMFIyZWtGQ1EyZFJjVVJEYUcxYVZFcHBXV3BaTWxwdFZteE9iVTE1Q2xsVVl6TmFWMDVxV21wQk5GbFVRVEZOUkdocldYcFZNMDVFU1RSYVJGRTBUMGRXYTAxQ2MwZERhWE5IUVZGUlFtYzNPSGRCVVhORlJGRjNUR015Vm5NS1dta3hiMkl6VGpCYVYxRjNVVUZaUzB0M1dVSkNRVWRFZG5wQlFrUkJVWGxFUkVKdlpFaFNkMk42YjNaTU1tUndaRWRvTVZscE5XcGlNakIyWVVoV2JncGFNbXgxV2pKYWFGa3lWWFpoTWxaNVltMVdjMk41TVdwaU1qRjBaRmMxY0dSSWEzZFBRVmxMUzNkWlFrSkJSMFIyZWtGQ1JGRlJjVVJEYUcxYVZFcHBDbGxxV1RKYWJWWnNUbTFOZVZsVVl6TmFWMDVxV21wQk5GbFVRVEZOUkdocldYcFZNMDVFU1RSYVJGRTBUMGRXYTAxQ09FZERhWE5IUVZGUlFtYzNPSGNLUVZFMFJVVlJkMUJqYlZadFkzazViMXBYUm10amVUbDBXVmRzZFUxQ2IwZERhWE5IUVZGUlFtYzNPSGRCVVRoRlJFRjNTMDFVUVROTlZGRXpUbFJWZVFwUFZFRjFRbWR2Y2tKblJVVkJXVTh2VFVGRlVVSkRRVTFJYldnd1pFaENlazlwT0haYU1td3dZVWhXYVV4dFRuWmlVemx2WkZka2JtRlhOVzVhYlVacUNscFVRVmxDWjI5eVFtZEZSVUZaVHk5TlFVVlNRa0Z2VFVORVNURk9la2wzVG5wUmVrMUhNRWREYVhOSFFWRlJRbWMzT0hkQlVrbEZXSGQ0WkdGSVVqQUtZMGhOTmt4NU9XNWhXRkp2WkZkSmRWa3lPWFJNTW1neFdqSmtjR0p0WkcxWlYwNXNUREowYkdOdE5XeGlTRTEwV1RJNWRHSllWblZoV0ZJMVRIazFiZ3BoV0ZKdlpGZEpkbVF5T1hsaE1scHpZak5rZWt3eVNqRmhWM2hyVEc1c2FHSlhlRUZqYlZadFkzazViMXBYUm10amVUbDBXVmRzZFUxRVowZERhWE5IQ2tGUlVVSm5OemgzUVZKTlJVdG5kMjlhYlZWNVdXMUpNazV0V214YVZGcHFUVzFGTTA0eVZtcFpNbGwzVDBkRmQwNVVRVFJhUjAweFRucFJlVTlIVVRBS1QwUm9iRnBFUVdoQ1oyOXlRbWRGUlVGWlR5OU5RVVZWUWtKTlRVVllaSFpqYlhSdFlrYzVNMWd5VW5Cak0wSm9aRWRPYjAxSFVVZERhWE5IUVZGUlFncG5OemgzUVZKVlJWWm5lRlZoU0ZJd1kwaE5Oa3g1T1c1aFdGSnZaRmRKZFZreU9YUk1NbWd4V2pKa2NHSnRaRzFaVjA1c1RESjBiR050Tld4aVNFMTBDbGt5T1hSaVdGWjFZVmhTTlV3eVJtcGtSMngyWW01TmRtTnVWblZqZVRoNlQxUkplazlVVlhoUFJFVjNUa001YUdSSVVteGlXRUl3WTNrNGVFMUNXVWNLUTJselIwRlJVVUpuTnpoM1FWSlpSVU5CZDBkalNGWnBZa2RzYWsxRldVZERhWE5IUVZGUlFtYzNPSGRCVW1kRlQwRjNNbU50Vm5kaWVuQnZaRmRrYmdwaFZ6VnVXbTFHYWxwVE9YSmFXRXAxV2xkNGVreFhUblppVnpFeFltMXNNR1ZVY0hsYVYxazJZMjFXYldONU9XOWFWMFpyWTNrNWRGbFhiSFZOU1VkTUNrSm5iM0pDWjBWRlFXUmFOVUZuVVVOQ1NEQkZaWGRDTlVGSVkwRXpWREIzWVhOaVNFVlVTbXBIVWpSamJWZGpNMEZ4U2t0WWNtcGxVRXN6TDJnMGNIa0taME00Y0Rkdk5FRkJRVWRvVm10d1JuTm5RVUZDUVUxQlUwUkNSMEZwUlVGek9XTlRVREJRTDNCU1dqUndjbUZpVGs1SmNUZzVWRk13VW1oWU1tWXpTZ3BTY0VoeWVXVkphRkl6V1VOSlVVUnVVWFYxV0hoVGJtNXVabXc0VVRkVlVrSlROMFJETWpOWmNrUmhVM0J6YUZOU1ZVVk9XSFJvVTJWNlFVdENaMmR4Q21ocmFrOVFVVkZFUVhkT2JrRkVRbXRCYWtWQk5HZERRMlV2TURCRVMxSXZOemhHYTNOU2N6RXhRbWROVjFOVGMybHphemhOUlRaUFRYbDRNbHBQUzBrS2RrMVdlVEZXU0VreE5VVTJXR1p5TTJselFTdEJhVGxhU1RaUGRIWTJRVzVQVm5jdmVsVklTMHgyZFdvMlZtOVpaRXh3TWpaVWQyaFhXVTR5VGpoTVJncGFhbnBXUjA1WUt6Qk9ZMDQ0VHpaeVdXTlFWWE4zUFQwS0xTMHRMUzFGVGtRZ1EwVlNWRWxHU1VOQlZFVXRMUzB0TFFvPSJ9fX19"}], "timestampVerificationData":{"rfc3161Timestamps":[{"signedTimestamp":"MIICyDADAgEAMIICvwYJKoZIhvcNAQcCoIICsDCCAqwCAQMxDTALBglghkgBZQMEAgEwgbcGCyqGSIb3DQEJEAEEoIGnBIGkMIGhAgEBBgkrBgEEAYO/MAIwMTANBglghkgBZQMEAgEFAAQgVRiPQiyjqHesLhjA1G/TxjESiy/FW8Sle/1OYNu3K/kCFAwvXO0pPiaW4uamLJqBsr3bixmnGA8yMDI2MTAxOTIyMzExMFowAwIBAaAypDAwLjEVMBMGA1UEChMMc2lnc3RvcmUuZGV2MRUwEwYDVQQDEwxzaWdzdG9yZS10c2GgADGCAdowggHWAgEBMFEwOTEVMBMGA1UEChMMc2lnc3RvcmUuZGV2MSAwHgYDVQQDExdzaWdzdG9yZS10c2Etc2VsZnNpZ25lZAIUOhNULwyQYe68wUMvy4qOiyojiwwwCwYJYIZIAWUDBAIBoIH8MBoGCSqGSIb3DQEJAzENBgsqhkiG9w0BCRABBDAcBgkqhkiG9w0BCQUxDxcNMjYxMDE5MjIzMTEwWjAvBgkqhkiG9w0BCQQxIgQgMtOdPWovwN2MtCUsGr55eXS/sj80XwnkzhWzvp3Z+ZYwgY4GCyqGSIb3DQEJEAIvMX8wfTB7MHkEIIX5J7wHq2LKw7RDVsEO/IGyxog/2nq55thw2dE6zQW3MFUwPaQ7MDkxFTATBgNVBAoTDHNpZ3N0b3JlLmRldjEgMB4GA1UEAxMXc2lnc3RvcmUtdHNhLXNlbGZzaWduZWQCFDoTVC8MkGHuvMFDL8uKjosqI4sMMAoGCCqGSM49BAMCBGYwZAIwa5hPScsoY8ZaoBk5cLEAUwhE+yEEOwm9xprfopTKz7qKFY1O9DeK5h6/gp2k6NtgAjByqi8mda8A+OGfG6rtkFHgjYnT6P/VUqQ95DNedeBl+vbMOity7vq28Gh+8O2yqBc="}]}}, "messageSignature":{"messageDigest":{"algorithm":"SHA2_256", "digest":"CPoM1QqE+UEnbF5tX6SfHU4B4VmHpgqrjyq24xnux3s="}, "signature":"MEUCICPSiaX9pY/oZbzuasGAnOQ0uL8P/JLHK+FKmUHYfvT2AiEAs8FC8HRyVEfj7AGjbsI6VfoahccKXs4FCfcyngTsQXc="}} \ No newline at end of file diff --git a/build/torch210-rocm70-x86_64-linux/mha.py b/build/torch210-rocm70-x86_64-linux/mha.py new file mode 100644 index 0000000000000000000000000000000000000000..01e2d6a0fbee98d27129f2eb695ac38df73e6487 --- /dev/null +++ b/build/torch210-rocm70-x86_64-linux/mha.py @@ -0,0 +1,332 @@ +"""FlashAttention forward entry points backed by aiter's Composable-Kernel FMHA. + +This is the compiled (HIP / Composable Kernel) counterpart to the Triton +``aiter-flash-attn`` package. Only the *forward* pass is provided (dense and +variable-length), including support for learnable attention sinks via +``window_size[2]`` / ``s_aux`` (e.g. gpt-oss). The op set ships exclusively +the CK ``mha_fwd`` / ``mha_varlen_fwd`` kernels, so the sink path is always +taken (no fall-through to a non-sink ASM kernel). +""" + +from typing import Optional, Tuple, Union + +import torch +from torch import Tensor + +from ._ops import ops + + +def _round_multiple(x: int, m: int) -> int: + return (x + m - 1) // m * m + + +def _maybe_pad_headdim(t: Tensor, target: int) -> Tensor: + og = t.size(-1) + if og == target: + return t + return torch.nn.functional.pad(t, [0, target - og]) + + +def _unpack_window(window_size) -> Tuple[int, int, int]: + left = int(window_size[0]) + right = int(window_size[1]) + sink = int(window_size[2]) if len(window_size) == 3 else 0 + return left, right, sink + + +def _prep_sink(s_aux: Optional[Tensor], nheads: int, device) -> Optional[Tensor]: + if s_aux is None: + return None + assert s_aux.device == device, "s_aux must be on the same device as q" + assert s_aux.shape[0] == nheads, "s_aux must have shape [nheads_q]" + if s_aux.dtype != torch.float32: + s_aux = s_aux.to(torch.float32) + return s_aux + + +def _effective_sink_size(sink_size: int, s_aux: Optional[Tensor]) -> int: + # The CK kernel only takes the sink path when sink_size > 0. Callers using + # the transformers attention interface pass the per-head sink logits as + # ``s_aux`` but leave ``window_size`` a 2-tuple (sink_size == 0), so enable + # a single learnable sink automatically when s_aux is given. + if s_aux is not None and sink_size == 0: + return 1 + return sink_size + + +def flash_attn_func( + q: Tensor, + k: Tensor, + v: Tensor, + dropout_p: float = 0.0, + softmax_scale: Optional[float] = None, + causal: bool = False, + window_size: Tuple[int, ...] = (-1, -1, 0), + bias: Optional[Tensor] = None, + alibi_slopes: Optional[Tensor] = None, + return_lse: bool = False, + return_attn_probs: bool = False, + s_aux: Optional[Tensor] = None, + **kwargs, +) -> Union[Tensor, Tuple[Tensor, ...]]: + """Dense FlashAttention forward. + + ``q, k, v`` have shape ``(batch, seqlen, nheads, headdim)``. ``window_size`` + is ``(left, right, sink_size)``; ``s_aux`` is an optional ``(nheads_q,)`` + fp32 tensor of learnable sink logits. Extra keyword arguments (e.g. those + passed by the transformers flash-attention interface) are ignored. + """ + if softmax_scale is None: + softmax_scale = q.shape[-1] ** (-0.5) + window_size_left, window_size_right, sink_size = _unpack_window(window_size) + + nheads_q = q.size(2) + s_aux = _prep_sink(s_aux, nheads_q, q.device) + sink_size = _effective_sink_size(sink_size, s_aux) + + head_q_og = q.size(-1) + head_v_og = v.size(-1) + head_q = _round_multiple(head_q_og, 8) + head_v = _round_multiple(head_v_og, 8) + q = _maybe_pad_headdim(q, head_q) + k = _maybe_pad_headdim(k, head_q) + v = _maybe_pad_headdim(v, head_v) + + return_softmax = return_attn_probs and dropout_p > 0 + out, softmax_lse, S_dmask, _ = ops.mha_fwd( + q, + k, + v, + dropout_p, + softmax_scale, + causal, + window_size_left, + window_size_right, + sink_size, + return_lse or return_attn_probs, + return_softmax, + None, # cu_seqlens_q + None, # cu_seqlens_kv + None, # out + bias, + alibi_slopes, + None, # q_descale + None, # k_descale + None, # v_descale + s_aux, + None, # gen + ) + + out = out[..., :head_v_og] + if not (return_lse or return_attn_probs): + return out + result = (out,) + if return_lse: + result = result + (softmax_lse,) + if return_attn_probs: + result = result + (S_dmask,) + return result + + +def flash_attn_varlen_func( + q: Tensor, + k: Tensor, + v: Tensor, + cu_seqlens_q: Tensor, + cu_seqlens_k: Tensor, + max_seqlen_q: int, + max_seqlen_k: int, + dropout_p: float = 0.0, + softmax_scale: Optional[float] = None, + causal: bool = False, + window_size: Tuple[int, ...] = (-1, -1, 0), + bias: Optional[Tensor] = None, + alibi_slopes: Optional[Tensor] = None, + return_lse: bool = False, + return_attn_probs: bool = False, + block_table: Optional[Tensor] = None, + min_seqlen_q: int = 0, + logits_soft_cap: float = 0.0, + zero_tensors: bool = False, + cu_seqlens_q_padded: Optional[Tensor] = None, + cu_seqlens_k_padded: Optional[Tensor] = None, + s_aux: Optional[Tensor] = None, + **kwargs, +) -> Union[Tensor, Tuple[Tensor, ...]]: + """Variable-length / packed FlashAttention forward. + + ``q`` has shape ``(total_q, nheads, headdim)``; ``cu_seqlens_*`` are int32 + cumulative offsets of shape ``(batch + 1,)``. Sinks are supported through + ``window_size[2]`` / ``s_aux`` as in :func:`flash_attn_func`. Extra keyword + arguments (e.g. from the transformers flash-attention interface) are ignored. + """ + if softmax_scale is None: + softmax_scale = q.shape[-1] ** (-0.5) + window_size_left, window_size_right, sink_size = _unpack_window(window_size) + + nheads_q = q.size(1) + s_aux = _prep_sink(s_aux, nheads_q, q.device) + sink_size = _effective_sink_size(sink_size, s_aux) + + head_q_og = q.size(-1) + head_v_og = v.size(-1) + head_q = _round_multiple(head_q_og, 8) + head_v = _round_multiple(head_v_og, 8) + q = _maybe_pad_headdim(q, head_q) + k = _maybe_pad_headdim(k, head_q) + v = _maybe_pad_headdim(v, head_v) + + return_softmax = return_attn_probs and dropout_p > 0 + out, softmax_lse, S_dmask, _ = ops.mha_varlen_fwd( + q, + k, + v, + cu_seqlens_q, + cu_seqlens_k, + max_seqlen_q, + max_seqlen_k, + min_seqlen_q, + dropout_p, + softmax_scale, + logits_soft_cap, + zero_tensors, + causal, + window_size_left, + window_size_right, + sink_size, + return_lse or return_attn_probs, + return_softmax, + None, # out + block_table, + bias, + alibi_slopes, + None, # q_descale + None, # k_descale + None, # v_descale + None, # gen + cu_seqlens_q_padded, + cu_seqlens_k_padded, + s_aux, + ) + + out = out[..., :head_v_og] + if not (return_lse or return_attn_probs): + return out + result = (out,) + if return_lse: + result = result + (softmax_lse,) + if return_attn_probs: + result = result + (S_dmask,) + return result + + +def _append_to_contiguous_cache(k_cache, v_cache, k, v, cache_seqlens, Sq): + """In-place append of new k/v into a contiguous cache at per-batch offsets.""" + B = k_cache.shape[0] + if isinstance(cache_seqlens, int): + cache_seqlens = torch.full((B,), cache_seqlens, device=k_cache.device, dtype=torch.long) + pos = cache_seqlens.view(B, 1).to(torch.long) + torch.arange(Sq, device=k_cache.device).view(1, Sq) + bidx = torch.arange(B, device=k_cache.device).view(B, 1).expand(B, Sq) + k_cache[bidx, pos] = k.to(k_cache.dtype) + v_cache[bidx, pos] = v.to(v_cache.dtype) + + +def flash_attn_with_kvcache( + q: Tensor, + k_cache: Tensor, + v_cache: Tensor, + k: Optional[Tensor] = None, + v: Optional[Tensor] = None, + rotary_cos: Optional[Tensor] = None, + rotary_sin: Optional[Tensor] = None, + cache_seqlens=None, + block_table: Optional[Tensor] = None, + softmax_scale: Optional[float] = None, + causal: bool = True, + window_size: Tuple[int, ...] = (-1, -1, 0), + s_aux: Optional[Tensor] = None, + return_lse: bool = False, + **kwargs, +) -> Union[Tensor, Tuple[Tensor, ...]]: + """FlashAttention against a KV cache, built on the CK split-KV kernel. + + ``q`` is ``(batch, seqlen_q, nheads, headdim)``. For a contiguous cache, + ``k_cache``/``v_cache`` are ``(batch, seqlen_cache, nheads_k, headdim)`` and + ``cache_seqlens`` (int or ``(batch,)``) is the number of valid cache entries + *before* appending. If ``k``/``v`` (the new tokens) are given, they are + written into the cache in place. The query attends to the valid prefix + (``cache_seqlens`` + ``seqlen_q``) with bottom-right causal alignment. + + Notes / current limitations: + - Rotary embedding is not applied here; pass ``q``/``k`` already rotated + (``rotary_cos``/``rotary_sin`` are rejected). + - Paged caches are addressed via ``block_table`` (forwarded to the split-KV + varlen kernel); in-cache append for paged layouts must be done by the + caller (pass ``k=v=None``). + """ + if rotary_cos is not None or rotary_sin is not None: + raise NotImplementedError( + "flash_attn_with_kvcache does not apply rotary; rotate q/k beforehand." + ) + if softmax_scale is None: + softmax_scale = q.shape[-1] ** (-0.5) + window_size_left, window_size_right, sink_size = _unpack_window(window_size) + B, Sq, Hq, D = q.shape + s_aux = _prep_sink(s_aux, Hq, q.device) + sink_size = _effective_sink_size(sink_size, s_aux) + + paged = block_table is not None + + # ---- append new tokens into the cache (contiguous layout only) ---- + if k is not None and not paged: + assert cache_seqlens is not None, "cache_seqlens required to append into the cache" + _append_to_contiguous_cache(k_cache, v_cache, k, v, cache_seqlens, Sq) + elif k is not None and paged: + raise NotImplementedError( + "in-cache append for paged layouts is not supported; append externally " + "and call with k=v=None." + ) + + appended = 0 if k is None else Sq + + # ---- key lengths ---- + if cache_seqlens is None: + klens = torch.full((B,), k_cache.shape[1], device=q.device, dtype=torch.long) + elif isinstance(cache_seqlens, int): + klens = torch.full((B,), cache_seqlens + appended, device=q.device, dtype=torch.long) + else: + klens = cache_seqlens.to(q.device, torch.long) + appended + + uniform = bool((klens == klens[0]).all().item()) + + # ---- fast path: contiguous cache, uniform length -> dense kernel ---- + if not paged and uniform: + L = int(klens[0].item()) + return flash_attn_func( + q, k_cache[:, :L], v_cache[:, :L], + softmax_scale=softmax_scale, causal=causal, + window_size=(window_size_left, window_size_right, sink_size), + s_aux=s_aux, return_lse=return_lse, + ) + + # ---- general path: variable cache lengths -> split-KV varlen kernel ---- + cu_q = torch.arange(0, (B + 1) * Sq, Sq, device=q.device, dtype=torch.int32) + cu_k = torch.zeros(B + 1, device=q.device, dtype=torch.int32) + cu_k[1:] = torch.cumsum(klens, 0).to(torch.int32) + q_packed = q.reshape(B * Sq, Hq, D) + if paged: + k_packed, v_packed = k_cache, v_cache # kernel gathers via block_table + else: + k_packed = torch.cat([k_cache[b, : int(klens[b])] for b in range(B)], dim=0) + v_packed = torch.cat([v_cache[b, : int(klens[b])] for b in range(B)], dim=0) + + out = flash_attn_varlen_func( + q_packed, k_packed, v_packed, cu_q, cu_k, Sq, int(klens.max().item()), + softmax_scale=softmax_scale, causal=causal, + window_size=(window_size_left, window_size_right, sink_size), + block_table=block_table, s_aux=s_aux, return_lse=return_lse, + ) + if return_lse: + out, lse = out + return out.reshape(B, Sq, Hq, D), lse + return out.reshape(B, Sq, Hq, D) diff --git a/build/torch210-rocm70-x86_64-linux/symbols.json b/build/torch210-rocm70-x86_64-linux/symbols.json new file mode 100644 index 0000000000000000000000000000000000000000..e8348db4bd94712352e7fb3e83b1d94f47c2814b --- /dev/null +++ b/build/torch210-rocm70-x86_64-linux/symbols.json @@ -0,0 +1,349 @@ +{ + "schema_version": 1, + "module": "_aiter_flash_attn_ck_rocm_fe2bb66", + "functions": [ + { + "name": "flash_attn_func", + "qualname": "flash_attn_func", + "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha", + "kind": "function", + "doc": "Dense FlashAttention forward.\n\n``q, k, v`` have shape ``(batch, seqlen, nheads, headdim)``. ``window_size``\nis ``(left, right, sink_size)``; ``s_aux`` is an optional ``(nheads_q,)``\nfp32 tensor of learnable sink logits. Extra keyword arguments (e.g. those\npassed by the transformers flash-attention interface) are ignored.", + "signature": { + "parameters": [ + { + "name": "q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "v", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "dropout_p", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "0.0", + "annotation": "float" + }, + { + "name": "softmax_scale", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "float | None" + }, + { + "name": "causal", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "window_size", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "(-1, -1, 0)", + "annotation": "Tuple[int, ...]" + }, + { + "name": "bias", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "alibi_slopes", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "return_lse", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "return_attn_probs", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "s_aux", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "kwargs", + "kind": "VAR_KEYWORD", + "default": null, + "annotation": null + } + ], + "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]" + } + }, + { + "name": "flash_attn_varlen_func", + "qualname": "flash_attn_varlen_func", + "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha", + "kind": "function", + "doc": "Variable-length / packed FlashAttention forward.\n\n``q`` has shape ``(total_q, nheads, headdim)``; ``cu_seqlens_*`` are int32\ncumulative offsets of shape ``(batch + 1,)``. Sinks are supported through\n``window_size[2]`` / ``s_aux`` as in :func:`flash_attn_func`. Extra keyword\narguments (e.g. from the transformers flash-attention interface) are ignored.", + "signature": { + "parameters": [ + { + "name": "q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "v", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "cu_seqlens_q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "cu_seqlens_k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "max_seqlen_q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "int" + }, + { + "name": "max_seqlen_k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "int" + }, + { + "name": "dropout_p", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "0.0", + "annotation": "float" + }, + { + "name": "softmax_scale", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "float | None" + }, + { + "name": "causal", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "window_size", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "(-1, -1, 0)", + "annotation": "Tuple[int, ...]" + }, + { + "name": "bias", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "alibi_slopes", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "return_lse", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "return_attn_probs", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "block_table", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "min_seqlen_q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "0", + "annotation": "int" + }, + { + "name": "logits_soft_cap", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "0.0", + "annotation": "float" + }, + { + "name": "zero_tensors", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "cu_seqlens_q_padded", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "cu_seqlens_k_padded", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "s_aux", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "kwargs", + "kind": "VAR_KEYWORD", + "default": null, + "annotation": null + } + ], + "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]" + } + }, + { + "name": "flash_attn_with_kvcache", + "qualname": "flash_attn_with_kvcache", + "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha", + "kind": "function", + "doc": "FlashAttention against a KV cache, built on the CK split-KV kernel.\n\n``q`` is ``(batch, seqlen_q, nheads, headdim)``. For a contiguous cache,\n``k_cache``/``v_cache`` are ``(batch, seqlen_cache, nheads_k, headdim)`` and\n``cache_seqlens`` (int or ``(batch,)``) is the number of valid cache entries\n*before* appending. If ``k``/``v`` (the new tokens) are given, they are\nwritten into the cache in place. The query attends to the valid prefix\n(``cache_seqlens`` + ``seqlen_q``) with bottom-right causal alignment.\n\nNotes / current limitations:\n- Rotary embedding is not applied here; pass ``q``/``k`` already rotated\n (``rotary_cos``/``rotary_sin`` are rejected).\n- Paged caches are addressed via ``block_table`` (forwarded to the split-KV\n varlen kernel); in-cache append for paged layouts must be done by the\n caller (pass ``k=v=None``).", + "signature": { + "parameters": [ + { + "name": "q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "k_cache", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "v_cache", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "v", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "rotary_cos", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "rotary_sin", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "cache_seqlens", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": null + }, + { + "name": "block_table", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "softmax_scale", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "float | None" + }, + { + "name": "causal", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "True", + "annotation": "bool" + }, + { + "name": "window_size", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "(-1, -1, 0)", + "annotation": "Tuple[int, ...]" + }, + { + "name": "s_aux", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "return_lse", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "kwargs", + "kind": "VAR_KEYWORD", + "default": null, + "annotation": null + } + ], + "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]" + } + } + ], + "layers": [] +} diff --git a/build/torch210-rocm71-x86_64-linux/__init__.py b/build/torch210-rocm71-x86_64-linux/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..73acb58c1142a1ab2a6ed9d6f34bc6a53ab5e9b5 --- /dev/null +++ b/build/torch210-rocm71-x86_64-linux/__init__.py @@ -0,0 +1,27 @@ +"""AITER Composable-Kernel FlashAttention kernels for AMD ROCm (compiled HIP). + +Compiled (Composable Kernel / HIP) counterpart to the Triton-based +``aiter-flash-attn`` package, repackaged from the CK FMHA forward path of the +`ROCm/aiter `_ project as a self-contained +Hugging Face Hub kernel. Exposes the FlashAttention forward entry points with +learnable attention sink support (e.g. gpt-oss). +""" + +from .mha import ( + flash_attn_func, + flash_attn_varlen_func, + flash_attn_with_kvcache, +) + + +__kernel_metadata__ = { + "license": "mit", +} + + +__all__ = [ + "__kernel_metadata__", + "flash_attn_func", + "flash_attn_varlen_func", + "flash_attn_with_kvcache", +] diff --git a/build/torch210-rocm71-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so b/build/torch210-rocm71-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so new file mode 100644 index 0000000000000000000000000000000000000000..ac9a4f6039f97a181afb8dea79899a0529632233 --- /dev/null +++ b/build/torch210-rocm71-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:04aebc20ca3de88cd0cb137fba3aaf0aa8b1ccd16588df660c70c8491653f01c +size 204259752 diff --git a/build/torch210-rocm71-x86_64-linux/_ops.py b/build/torch210-rocm71-x86_64-linux/_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..2b4dd56c220b61fd2c16e0c1a02ee91a491cef0c --- /dev/null +++ b/build/torch210-rocm71-x86_64-linux/_ops.py @@ -0,0 +1,9 @@ +import torch +from . import _aiter_flash_attn_ck_rocm_fe2bb66 +ops = torch.ops._aiter_flash_attn_ck_rocm_fe2bb66 + +def add_op_namespace_prefix(op_name: str): + """ + Prefix op by namespace. + """ + return f"_aiter_flash_attn_ck_rocm_fe2bb66::{op_name}" diff --git a/build/torch210-rocm71-x86_64-linux/metadata.json b/build/torch210-rocm71-x86_64-linux/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..e1adf1cc7d2630060ea3b0ad4582d05c2ff33fd3 --- /dev/null +++ b/build/torch210-rocm71-x86_64-linux/metadata.json @@ -0,0 +1,37 @@ +{ + "name": "aiter-flash-attn-ck", + "id": "_aiter_flash_attn_ck_rocm_fe2bb66", + "version": 1, + "kernels-minver": "0.14.0", + "license": "MIT", + "python-depends": [], + "kernel-depends": [], + "backend": { + "type": "rocm", + "archs": [ + "gfx942", + "gfx950" + ] + }, + "digest": { + "algorithm": "sha256", + "files": { + "__init__.py": "7wj1GTm2bqTLLCyT4xLuAk/+JWYrhINjY/ODMNWhgZ0=", + "_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so": "BK68IMo96IzQyxN/ujqvCqixzNFliN9mDHDISRZT8Bw=", + "_ops.py": "4WkdYDWWJp60zep+oW/5ym9vIeVMNfrgRCkP3o/w7yU=", + "mha.py": "zCAjY4O+DiCzmEL+W9sqXxZaqjsj4RJbWFcyPpUZejk=", + "symbols.json": "rhnF4H8Jsb6I+YPxIVFMgnLcTQJODEXbZ7ge77NZXAU=" + } + }, + "provenance": { + "kernel-builder": { + "version": "0.18.0-dev0", + "commit": "d054906a13710b811b96697b4635f03020842733", + "dirty": false + }, + "kernel": { + "commit": "fe2bb66fee6c2a77eccf08a0508dc57428d488ed", + "dirty": false + } + } +} \ No newline at end of file diff --git a/build/torch210-rocm71-x86_64-linux/metadata.json.sigstore b/build/torch210-rocm71-x86_64-linux/metadata.json.sigstore new file mode 100644 index 0000000000000000000000000000000000000000..4f877d22525372e1623c4f463c4d968944f954aa --- /dev/null +++ b/build/torch210-rocm71-x86_64-linux/metadata.json.sigstore @@ -0,0 +1 @@ +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json", "verificationMaterial":{"certificate":{"rawBytes":"MIIHLDCCBrKgAwIBAgIUMp+DSFw+JgfWV7TcLB0c0Mb65MwwCgYIKoZIzj0EAwMwNzEVMBMGA1UEChMMc2lnc3RvcmUuZGV2MR4wHAYDVQQDExVzaWdzdG9yZS1pbnRlcm1lZGlhdGUwHhcNMjYxMDE5MjIzMTExWhcNMjYxMDE5MjI0MTExWjAAMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE8kLbyyRJUo+k0QdJbUOn+ybnTHhtfKEr7dpiN7ICGRBSv7puurajV7kDVk6RJbao8Okhui99gTQi7oGPT0Nxt6OCBdEwggXNMA4GA1UdDwEB/wQEAwIHgDATBgNVHSUEDDAKBggrBgEFBQcDAzAfBgNVHSMEGDAWgBTf0+nPViQRlvmo2OkoVaLGLhhkPzBrBgNVHREBAf8EYTBfhl1odHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkvLmdpdGh1Yi93b3JrZmxvd3MvYnVpbGQueWFtbEByZWZzL2hlYWRzL21haW4wOQYKKwYBBAGDvzABAQQraHR0cHM6Ly90b2tlbi5hY3Rpb25zLmdpdGh1YnVzZXJjb250ZW50LmNvbTAfBgorBgEEAYO/MAECBBF3b3JrZmxvd19kaXNwYXRjaDA2BgorBgEEAYO/MAEDBChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMBMGCisGAQQBg78wAQQEBUJ1aWxkMCsGCisGAQQBg78wAQUEHWh1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5MB0GCisGAQQBg78wAQYED3JlZnMvaGVhZHMvbWFpbjA7BgorBgEEAYO/MAEIBC0MK2h0dHBzOi8vdG9rZW4uYWN0aW9ucy5naXRodWJ1c2VyY29udGVudC5jb20wbQYKKwYBBAGDvzABCQRfDF1odHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkvLmdpdGh1Yi93b3JrZmxvd3MvYnVpbGQueWFtbEByZWZzL2hlYWRzL21haW4wOAYKKwYBBAGDvzABCgQqDChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMBsGCisGAQQBg78wAQsEDQwLc2VsZi1ob3N0ZWQwQAYKKwYBBAGDvzABDAQyDDBodHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkwOAYKKwYBBAGDvzABDQQqDChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMB8GCisGAQQBg78wAQ4EEQwPcmVmcy9oZWFkcy9tYWluMBoGCisGAQQBg78wAQ8EDAwKMTA3MTQ3NTUyOTAuBgorBgEEAYO/MAEQBCAMHmh0dHBzOi8vZ2l0aHViLmNvbS9odWdnaW5nZmFjZTAYBgorBgEEAYO/MAERBAoMCDI1NzIwNzQzMG0GCisGAQQBg78wARIEXwxdaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5Ly5naXRodWIvd29ya2Zsb3dzL2J1aWxkLnlhbWxAcmVmcy9oZWFkcy9tYWluMDgGCisGAQQBg78wARMEKgwoZmUyYmI2NmZlZTZjMmE3N2VjY2YwOGEwNTA4ZGM1NzQyOGQ0ODhlZDAhBgorBgEEAYO/MAEUBBMMEXdvcmtmbG93X2Rpc3BhdGNoMGQGCisGAQQBg78wARUEVgxUaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5L2FjdGlvbnMvcnVucy8zOTIzOTUxODEwNC9hdHRlbXB0cy8xMBYGCisGAQQBg78wARYECAwGcHVibGljMEYGCisGAQQBg78wARgEOAw2cmVwbzpodWdnaW5nZmFjZS9rZXJuZWxzLWNvbW11bml0eTpyZWY6cmVmcy9oZWFkcy9tYWluMIGKBgorBgEEAdZ5AgQCBHwEegB4AHYA3T0wasbHETJjGR4cmWc3AqJKXrjePK3/h4pygC8p7o4AAAGhVkpJ0wAABAMARzBFAiBd9Y7k3OakcDXOUOtPyvjvTHZaUz/B/tF4M/lrAPbSkgIhAMS6DTW0k6oKIEa1SATYHGWoTmXaF3t6lEktxjLrj5OMMAoGCCqGSM49BAMDA2gAMGUCMH23PJfryGMZHm69vca7K+4CeSSdtIsPRohlbI62CVTQgx/5zELDp54UGy2s+s8O0AIxAIDnrnYiN4CaOuqRlp6WpPLQuG0Xvzk3XM/ioaBbCV9mGfH1y674/krGMefcljicUw=="}, "tlogEntries":[{"logIndex":"3327211242", "logId":{"keyId":"wNI9atQGlz+VWfO6LRygH4QUfY/8W4RFwiT5i5WRgB0="}, "kindVersion":{"kind":"hashedrekord", "version":"0.0.1"}, "integratedTime":"1792449071", "inclusionPromise":{"signedEntryTimestamp":"MEQCIAr6PySJYJqy9DPHTtQgj9lSQLP+7jzp1PQhaQdDBhOkAiATFyyPYAtt+8DMi8K3HCc98mwBas4Kn6GOtrmIo+u8cA=="}, "inclusionProof":{"logIndex":"3205306980", "rootHash":"/n2pIUbsEGrPvDVhc62xJVK1pGu6X6FIjXWKAB9B11s=", "treeSize":"3205307004", "hashes":["b8JPr2VEt72nRhnrDm3LGBUw57uu9LLjYUP7BrgM9j8=", "uj8iz9+habwy51518PtXayHwhXS1UVWHasW4Dw4HEv8=", "ZzHzDD4a8QdIuk81wimU52tqU/HqEaVUd5zE4szbH5k=", "hKyjtQs1OpKphopD5PuSJ+QLeWBbsxl1l8V8vcQPgug=", "GNln8dir9ny0f06ahhipF8UWPa4XL9o9Bf1LV9qfH9M=", "v5NV1PqmPcm+8RACz+834UW43Jvx9T42P+ziCOAZqEY=", "t5D+40j8Z6l/36YGQvBik1shdJk9qiXxj5J6yRy+Ilw=", "t4Yy4kaoiJ3+VosqXw5Ogmg+A7p/E6y8+JhqWbalMbI=", "5egUlZ4Pi0Ix/qLyCDXocsWywJCHqrde/S06vOZvlOM=", "QsXufUpoKtSjKAB19K1G00umQpWvPHlBNuRGFiiVjQQ=", "QIMkwQbwAewhLJd//5FFwI3SnyNQBGjCgJrelfG8vhc=", "Et17CACUv9QN64irUJuBjsrgRGkGnnJ9xacRV/aBdwQ=", "lqgEwkxhDMs/gdIAE4lmjfMwg7nDmv7lda6O0LGliCw=", "HovTPBWzoErvUg1u5sIi3x1ZonZ7yJekVsBrmpomCIA=", "G+nk7mgE5h6dUmv2UY7NDJrY3WvrZY31raaO1AETt+o=", "qlTWNDzERnTBQVcHHXXVPjiUP0SsikGxfcxVlgqVFDo=", "gg9GNApNke3+zNhl4tsWtmIycCFrOMCB3lys6XuSQNQ=", "NlOclldmKPA1tMKIskqHAwuQDYHp31YwpK+OOMrMccc=", "qxzHanAzz57SDdmJe0B7bJK72NTIBbwEMGBBKvDOROw=", "xH/DCseLHr9eKoYT8qsORZK7zVdEGYWHuVtsVrD95wY="], "checkpoint":{"envelope":"rekor.sigstore.dev - 1193050959916656506\n3205307004\n/n2pIUbsEGrPvDVhc62xJVK1pGu6X6FIjXWKAB9B11s=\n\n— rekor.sigstore.dev wNI9ajBFAiAbIQNvzdYi+kJoZke0K/Pqgv3/lAKAHbSNYL82CNBKXQIhAJl7oEPKWHot9DzcEzfhRkqZL/pbPqMksdsMLXmuh/4s\n"}}, "canonicalizedBody":"eyJhcGlWZXJzaW9uIjoiMC4wLjEiLCJraW5kIjoiaGFzaGVkcmVrb3JkIiwic3BlYyI6eyJkYXRhIjp7Imhhc2giOnsiYWxnb3JpdGhtIjoic2hhMjU2IiwidmFsdWUiOiI5MWY4ZDQxN2NiYzEyMDY4YjIyMGQwNmY4ZmYxNTg5MzBiNTdjZTc1ZjVjMzkzOTc5MTQwOTgwNzQ1YThmNTQyIn19LCJzaWduYXR1cmUiOnsiY29udGVudCI6Ik1FUUNJR0hqWFJuSmpUbnFaaVZ1QmpNZU9oYU54aHIwSFE2enpSckI0cVBlN1Y5M0FpQm9QeS9kT1FLcVZNOWlnTGpSbmhOeFFBQ1plLzM4YWZLSFBqMExNZk5NdUE9PSIsInB1YmxpY0tleSI6eyJjb250ZW50IjoiTFMwdExTMUNSVWRKVGlCRFJWSlVTVVpKUTBGVVJTMHRMUzB0Q2sxSlNVaE1SRU5EUW5KTFowRjNTVUpCWjBsVlRYQXJSRk5HZHl0S1oyWlhWamRVWTB4Q01HTXdUV0kyTlUxM2QwTm5XVWxMYjFwSmVtb3dSVUYzVFhjS1RucEZWazFDVFVkQk1WVkZRMmhOVFdNeWJHNWpNMUoyWTIxVmRWcEhWakpOVWpSM1NFRlpSRlpSVVVSRmVGWjZZVmRrZW1SSE9YbGFVekZ3WW01U2JBcGpiVEZzV2tkc2FHUkhWWGRJYUdOT1RXcFplRTFFUlRWTmFrbDZUVlJGZUZkb1kwNU5hbGw0VFVSRk5VMXFTVEJOVkVWNFYycEJRVTFHYTNkRmQxbElDa3R2V2tsNmFqQkRRVkZaU1V0dldrbDZhakJFUVZGalJGRm5RVVU0YTB4aWVYbFNTbFZ2SzJzd1VXUktZbFZQYml0NVltNVVTR2gwWmt0RmNqZGtjR2tLVGpkSlEwZFNRbE4yTjNCMWRYSmhhbFkzYTBSV2F6WlNTbUpoYnpoUGEyaDFhVGs1WjFSUmFUZHZSMUJVTUU1NGREWlBRMEprUlhkbloxaE9UVUUwUndwQk1WVmtSSGRGUWk5M1VVVkJkMGxJWjBSQlZFSm5UbFpJVTFWRlJFUkJTMEpuWjNKQ1owVkdRbEZqUkVGNlFXWkNaMDVXU0ZOTlJVZEVRVmRuUWxSbUNqQXJibEJXYVZGU2JIWnRiekpQYTI5V1lVeEhUR2hvYTFCNlFuSkNaMDVXU0ZKRlFrRm1PRVZaVkVKbWFHd3hiMlJJVW5kamVtOTJUREprY0dSSGFERUtXV2sxYW1JeU1IWmhTRlp1V2pKc2RWb3lXbWhaTWxWMllUSldlV0p0Vm5OamVURnFZakl4ZEdSWE5YQmtTR3QyVEcxa2NHUkhhREZaYVRrellqTktjZ3BhYlhoMlpETk5kbGx1Vm5CaVIxRjFaVmRHZEdKRlFubGFWMXA2VERKb2JGbFhVbnBNTWpGb1lWYzBkMDlSV1V0TGQxbENRa0ZIUkhaNlFVSkJVVkZ5Q21GSVVqQmpTRTAyVEhrNU1HSXlkR3hpYVRWb1dUTlNjR0l5TlhwTWJXUndaRWRvTVZsdVZucGFXRXBxWWpJMU1GcFhOVEJNYlU1MllsUkJaa0puYjNJS1FtZEZSVUZaVHk5TlFVVkRRa0pHTTJJelNuSmFiWGgyWkRFNWEyRllUbmRaV0ZKcVlVUkJNa0puYjNKQ1owVkZRVmxQTDAxQlJVUkNRMmh0V2xSS2FRcFphbGt5V20xV2JFNXRUWGxaVkdNeldsZE9hbHBxUVRSWlZFRXhUVVJvYTFsNlZUTk9SRWswV2tSUk5FOUhWbXROUWsxSFEybHpSMEZSVVVKbk56aDNDa0ZSVVVWQ1ZVb3hZVmQ0YTAxRGMwZERhWE5IUVZGUlFtYzNPSGRCVVZWRlNGZG9NVm95WkhCaWJXUnRXVmRPYkV3eWRHeGpiVFZzWWtoTmRGa3lPWFFLWWxoV2RXRllValZOUWpCSFEybHpSMEZSVVVKbk56aDNRVkZaUlVRelNteGFiazEyWVVkV2FGcElUWFppVjBad1ltcEJOMEpuYjNKQ1owVkZRVmxQTHdwTlFVVkpRa013VFVzeWFEQmtTRUo2VDJrNGRtUkhPWEphVnpSMVdWZE9NR0ZYT1hWamVUVnVZVmhTYjJSWFNqRmpNbFo1V1RJNWRXUkhWblZrUXpWcUNtSXlNSGRpVVZsTFMzZFpRa0pCUjBSMmVrRkNRMUZTWmtSR01XOWtTRkozWTNwdmRrd3laSEJrUjJneFdXazFhbUl5TUhaaFNGWnVXakpzZFZveVdtZ0tXVEpWZG1FeVZubGliVlp6WTNreGFtSXlNWFJrVnpWd1pFaHJka3h0WkhCa1IyZ3hXV2s1TTJJelNuSmFiWGgyWkROTmRsbHVWbkJpUjFGMVpWZEdkQXBpUlVKNVdsZGFla3d5YUd4WlYxSjZUREl4YUdGWE5IZFBRVmxMUzNkWlFrSkJSMFIyZWtGQ1EyZFJjVVJEYUcxYVZFcHBXV3BaTWxwdFZteE9iVTE1Q2xsVVl6TmFWMDVxV21wQk5GbFVRVEZOUkdocldYcFZNMDVFU1RSYVJGRTBUMGRXYTAxQ2MwZERhWE5IUVZGUlFtYzNPSGRCVVhORlJGRjNUR015Vm5NS1dta3hiMkl6VGpCYVYxRjNVVUZaUzB0M1dVSkNRVWRFZG5wQlFrUkJVWGxFUkVKdlpFaFNkMk42YjNaTU1tUndaRWRvTVZscE5XcGlNakIyWVVoV2JncGFNbXgxV2pKYWFGa3lWWFpoTWxaNVltMVdjMk41TVdwaU1qRjBaRmMxY0dSSWEzZFBRVmxMUzNkWlFrSkJSMFIyZWtGQ1JGRlJjVVJEYUcxYVZFcHBDbGxxV1RKYWJWWnNUbTFOZVZsVVl6TmFWMDVxV21wQk5GbFVRVEZOUkdocldYcFZNMDVFU1RSYVJGRTBUMGRXYTAxQ09FZERhWE5IUVZGUlFtYzNPSGNLUVZFMFJVVlJkMUJqYlZadFkzazViMXBYUm10amVUbDBXVmRzZFUxQ2IwZERhWE5IUVZGUlFtYzNPSGRCVVRoRlJFRjNTMDFVUVROTlZGRXpUbFJWZVFwUFZFRjFRbWR2Y2tKblJVVkJXVTh2VFVGRlVVSkRRVTFJYldnd1pFaENlazlwT0haYU1td3dZVWhXYVV4dFRuWmlVemx2WkZka2JtRlhOVzVhYlVacUNscFVRVmxDWjI5eVFtZEZSVUZaVHk5TlFVVlNRa0Z2VFVORVNURk9la2wzVG5wUmVrMUhNRWREYVhOSFFWRlJRbWMzT0hkQlVrbEZXSGQ0WkdGSVVqQUtZMGhOTmt4NU9XNWhXRkp2WkZkSmRWa3lPWFJNTW1neFdqSmtjR0p0WkcxWlYwNXNUREowYkdOdE5XeGlTRTEwV1RJNWRHSllWblZoV0ZJMVRIazFiZ3BoV0ZKdlpGZEpkbVF5T1hsaE1scHpZak5rZWt3eVNqRmhWM2hyVEc1c2FHSlhlRUZqYlZadFkzazViMXBYUm10amVUbDBXVmRzZFUxRVowZERhWE5IQ2tGUlVVSm5OemgzUVZKTlJVdG5kMjlhYlZWNVdXMUpNazV0V214YVZGcHFUVzFGTTA0eVZtcFpNbGwzVDBkRmQwNVVRVFJhUjAweFRucFJlVTlIVVRBS1QwUm9iRnBFUVdoQ1oyOXlRbWRGUlVGWlR5OU5RVVZWUWtKTlRVVllaSFpqYlhSdFlrYzVNMWd5VW5Cak0wSm9aRWRPYjAxSFVVZERhWE5IUVZGUlFncG5OemgzUVZKVlJWWm5lRlZoU0ZJd1kwaE5Oa3g1T1c1aFdGSnZaRmRKZFZreU9YUk1NbWd4V2pKa2NHSnRaRzFaVjA1c1RESjBiR050Tld4aVNFMTBDbGt5T1hSaVdGWjFZVmhTTlV3eVJtcGtSMngyWW01TmRtTnVWblZqZVRoNlQxUkplazlVVlhoUFJFVjNUa001YUdSSVVteGlXRUl3WTNrNGVFMUNXVWNLUTJselIwRlJVVUpuTnpoM1FWSlpSVU5CZDBkalNGWnBZa2RzYWsxRldVZERhWE5IUVZGUlFtYzNPSGRCVW1kRlQwRjNNbU50Vm5kaWVuQnZaRmRrYmdwaFZ6VnVXbTFHYWxwVE9YSmFXRXAxV2xkNGVreFhUblppVnpFeFltMXNNR1ZVY0hsYVYxazJZMjFXYldONU9XOWFWMFpyWTNrNWRGbFhiSFZOU1VkTENrSm5iM0pDWjBWRlFXUmFOVUZuVVVOQ1NIZEZaV2RDTkVGSVdVRXpWREIzWVhOaVNFVlVTbXBIVWpSamJWZGpNMEZ4U2t0WWNtcGxVRXN6TDJnMGNIa0taME00Y0Rkdk5FRkJRVWRvVm10d1NqQjNRVUZDUVUxQlVucENSa0ZwUW1RNVdUZHJNMDloYTJORVdFOVZUM1JRZVhacWRsUklXbUZWZWk5Q0wzUkdOQXBOTDJ4eVFWQmlVMnRuU1doQlRWTTJSRlJYTUdzMmIwdEpSV0V4VTBGVVdVaEhWMjlVYlZoaFJqTjBObXhGYTNSNGFreHlhalZQVFUxQmIwZERRM0ZIQ2xOTk5EbENRVTFFUVRKblFVMUhWVU5OU0RJelVFcG1jbmxIVFZwSWJUWTVkbU5oTjBzck5FTmxVMU5rZEVselVGSnZhR3hpU1RZeVExWlVVV2Q0THpVS2VrVk1SSEExTkZWSGVUSnpLM000VHpCQlNYaEJTVVJ1Y201WmFVNDBRMkZQZFhGU2JIQTJWM0JRVEZGMVJ6QllkbnByTTFoTkwybHZZVUppUTFZNWJRcEhaa2d4ZVRZM05DOXJja2ROWldaamJHcHBZMVYzUFQwS0xTMHRMUzFGVGtRZ1EwVlNWRWxHU1VOQlZFVXRMUzB0TFFvPSJ9fX19"}], "timestampVerificationData":{"rfc3161Timestamps":[{"signedTimestamp":"MIICyzADAgEAMIICwgYJKoZIhvcNAQcCoIICszCCAq8CAQMxDTALBglghkgBZQMEAgEwgbgGCyqGSIb3DQEJEAEEoIGoBIGlMIGiAgEBBgkrBgEEAYO/MAIwMTANBglghkgBZQMEAgEFAAQghhLtVKNwP9C9AoYe4HEUs+7awj5hU/huvyNh5KwKZwoCFQDsr/tPNk6bbSMFPHHhg8PZMEWfhxgPMjAyNjEwMTkyMjMxMTFaMAMCAQGgMqQwMC4xFTATBgNVBAoTDHNpZ3N0b3JlLmRldjEVMBMGA1UEAxMMc2lnc3RvcmUtdHNhoAAxggHcMIIB2AIBATBRMDkxFTATBgNVBAoTDHNpZ3N0b3JlLmRldjEgMB4GA1UEAxMXc2lnc3RvcmUtdHNhLXNlbGZzaWduZWQCFDoTVC8MkGHuvMFDL8uKjosqI4sMMAsGCWCGSAFlAwQCAaCB/DAaBgkqhkiG9w0BCQMxDQYLKoZIhvcNAQkQAQQwHAYJKoZIhvcNAQkFMQ8XDTI2MTAxOTIyMzExMVowLwYJKoZIhvcNAQkEMSIEIAVFYYjePZj3iCjN5TwF261bkicR+j86cqwhc/Ylrv1OMIGOBgsqhkiG9w0BCRACLzF/MH0wezB5BCCF+Se8B6tiysO0Q1bBDvyBssaIP9p6uebYcNnROs0FtzBVMD2kOzA5MRUwEwYDVQQKEwxzaWdzdG9yZS5kZXYxIDAeBgNVBAMTF3NpZ3N0b3JlLXRzYS1zZWxmc2lnbmVkAhQ6E1QvDJBh7rzBQy/Lio6LKiOLDDAKBggqhkjOPQQDAgRoMGYCMQCaOXxeRZ9FR864IHhKG6YZGWsx5NQvJZPbeYBSvP+tMDPnsAAdW0y0QUPJTGc2qLQCMQCaRwSvFIurXXHVpNG8n/+B9jzewECiPpGV67deqVZWywtL/TmaVHNu8YpEgd9efms="}]}}, "messageSignature":{"messageDigest":{"algorithm":"SHA2_256", "digest":"kfjUF8vBIGiyINBvj/FYkwtXznX1w5OXkUCYB0Wo9UI="}, "signature":"MEQCIGHjXRnJjTnqZiVuBjMeOhaNxhr0HQ6zzRrB4qPe7V93AiBoPy/dOQKqVM9igLjRnhNxQACZe/38afKHPj0LMfNMuA=="}} \ No newline at end of file diff --git a/build/torch210-rocm71-x86_64-linux/mha.py b/build/torch210-rocm71-x86_64-linux/mha.py new file mode 100644 index 0000000000000000000000000000000000000000..01e2d6a0fbee98d27129f2eb695ac38df73e6487 --- /dev/null +++ b/build/torch210-rocm71-x86_64-linux/mha.py @@ -0,0 +1,332 @@ +"""FlashAttention forward entry points backed by aiter's Composable-Kernel FMHA. + +This is the compiled (HIP / Composable Kernel) counterpart to the Triton +``aiter-flash-attn`` package. Only the *forward* pass is provided (dense and +variable-length), including support for learnable attention sinks via +``window_size[2]`` / ``s_aux`` (e.g. gpt-oss). The op set ships exclusively +the CK ``mha_fwd`` / ``mha_varlen_fwd`` kernels, so the sink path is always +taken (no fall-through to a non-sink ASM kernel). +""" + +from typing import Optional, Tuple, Union + +import torch +from torch import Tensor + +from ._ops import ops + + +def _round_multiple(x: int, m: int) -> int: + return (x + m - 1) // m * m + + +def _maybe_pad_headdim(t: Tensor, target: int) -> Tensor: + og = t.size(-1) + if og == target: + return t + return torch.nn.functional.pad(t, [0, target - og]) + + +def _unpack_window(window_size) -> Tuple[int, int, int]: + left = int(window_size[0]) + right = int(window_size[1]) + sink = int(window_size[2]) if len(window_size) == 3 else 0 + return left, right, sink + + +def _prep_sink(s_aux: Optional[Tensor], nheads: int, device) -> Optional[Tensor]: + if s_aux is None: + return None + assert s_aux.device == device, "s_aux must be on the same device as q" + assert s_aux.shape[0] == nheads, "s_aux must have shape [nheads_q]" + if s_aux.dtype != torch.float32: + s_aux = s_aux.to(torch.float32) + return s_aux + + +def _effective_sink_size(sink_size: int, s_aux: Optional[Tensor]) -> int: + # The CK kernel only takes the sink path when sink_size > 0. Callers using + # the transformers attention interface pass the per-head sink logits as + # ``s_aux`` but leave ``window_size`` a 2-tuple (sink_size == 0), so enable + # a single learnable sink automatically when s_aux is given. + if s_aux is not None and sink_size == 0: + return 1 + return sink_size + + +def flash_attn_func( + q: Tensor, + k: Tensor, + v: Tensor, + dropout_p: float = 0.0, + softmax_scale: Optional[float] = None, + causal: bool = False, + window_size: Tuple[int, ...] = (-1, -1, 0), + bias: Optional[Tensor] = None, + alibi_slopes: Optional[Tensor] = None, + return_lse: bool = False, + return_attn_probs: bool = False, + s_aux: Optional[Tensor] = None, + **kwargs, +) -> Union[Tensor, Tuple[Tensor, ...]]: + """Dense FlashAttention forward. + + ``q, k, v`` have shape ``(batch, seqlen, nheads, headdim)``. ``window_size`` + is ``(left, right, sink_size)``; ``s_aux`` is an optional ``(nheads_q,)`` + fp32 tensor of learnable sink logits. Extra keyword arguments (e.g. those + passed by the transformers flash-attention interface) are ignored. + """ + if softmax_scale is None: + softmax_scale = q.shape[-1] ** (-0.5) + window_size_left, window_size_right, sink_size = _unpack_window(window_size) + + nheads_q = q.size(2) + s_aux = _prep_sink(s_aux, nheads_q, q.device) + sink_size = _effective_sink_size(sink_size, s_aux) + + head_q_og = q.size(-1) + head_v_og = v.size(-1) + head_q = _round_multiple(head_q_og, 8) + head_v = _round_multiple(head_v_og, 8) + q = _maybe_pad_headdim(q, head_q) + k = _maybe_pad_headdim(k, head_q) + v = _maybe_pad_headdim(v, head_v) + + return_softmax = return_attn_probs and dropout_p > 0 + out, softmax_lse, S_dmask, _ = ops.mha_fwd( + q, + k, + v, + dropout_p, + softmax_scale, + causal, + window_size_left, + window_size_right, + sink_size, + return_lse or return_attn_probs, + return_softmax, + None, # cu_seqlens_q + None, # cu_seqlens_kv + None, # out + bias, + alibi_slopes, + None, # q_descale + None, # k_descale + None, # v_descale + s_aux, + None, # gen + ) + + out = out[..., :head_v_og] + if not (return_lse or return_attn_probs): + return out + result = (out,) + if return_lse: + result = result + (softmax_lse,) + if return_attn_probs: + result = result + (S_dmask,) + return result + + +def flash_attn_varlen_func( + q: Tensor, + k: Tensor, + v: Tensor, + cu_seqlens_q: Tensor, + cu_seqlens_k: Tensor, + max_seqlen_q: int, + max_seqlen_k: int, + dropout_p: float = 0.0, + softmax_scale: Optional[float] = None, + causal: bool = False, + window_size: Tuple[int, ...] = (-1, -1, 0), + bias: Optional[Tensor] = None, + alibi_slopes: Optional[Tensor] = None, + return_lse: bool = False, + return_attn_probs: bool = False, + block_table: Optional[Tensor] = None, + min_seqlen_q: int = 0, + logits_soft_cap: float = 0.0, + zero_tensors: bool = False, + cu_seqlens_q_padded: Optional[Tensor] = None, + cu_seqlens_k_padded: Optional[Tensor] = None, + s_aux: Optional[Tensor] = None, + **kwargs, +) -> Union[Tensor, Tuple[Tensor, ...]]: + """Variable-length / packed FlashAttention forward. + + ``q`` has shape ``(total_q, nheads, headdim)``; ``cu_seqlens_*`` are int32 + cumulative offsets of shape ``(batch + 1,)``. Sinks are supported through + ``window_size[2]`` / ``s_aux`` as in :func:`flash_attn_func`. Extra keyword + arguments (e.g. from the transformers flash-attention interface) are ignored. + """ + if softmax_scale is None: + softmax_scale = q.shape[-1] ** (-0.5) + window_size_left, window_size_right, sink_size = _unpack_window(window_size) + + nheads_q = q.size(1) + s_aux = _prep_sink(s_aux, nheads_q, q.device) + sink_size = _effective_sink_size(sink_size, s_aux) + + head_q_og = q.size(-1) + head_v_og = v.size(-1) + head_q = _round_multiple(head_q_og, 8) + head_v = _round_multiple(head_v_og, 8) + q = _maybe_pad_headdim(q, head_q) + k = _maybe_pad_headdim(k, head_q) + v = _maybe_pad_headdim(v, head_v) + + return_softmax = return_attn_probs and dropout_p > 0 + out, softmax_lse, S_dmask, _ = ops.mha_varlen_fwd( + q, + k, + v, + cu_seqlens_q, + cu_seqlens_k, + max_seqlen_q, + max_seqlen_k, + min_seqlen_q, + dropout_p, + softmax_scale, + logits_soft_cap, + zero_tensors, + causal, + window_size_left, + window_size_right, + sink_size, + return_lse or return_attn_probs, + return_softmax, + None, # out + block_table, + bias, + alibi_slopes, + None, # q_descale + None, # k_descale + None, # v_descale + None, # gen + cu_seqlens_q_padded, + cu_seqlens_k_padded, + s_aux, + ) + + out = out[..., :head_v_og] + if not (return_lse or return_attn_probs): + return out + result = (out,) + if return_lse: + result = result + (softmax_lse,) + if return_attn_probs: + result = result + (S_dmask,) + return result + + +def _append_to_contiguous_cache(k_cache, v_cache, k, v, cache_seqlens, Sq): + """In-place append of new k/v into a contiguous cache at per-batch offsets.""" + B = k_cache.shape[0] + if isinstance(cache_seqlens, int): + cache_seqlens = torch.full((B,), cache_seqlens, device=k_cache.device, dtype=torch.long) + pos = cache_seqlens.view(B, 1).to(torch.long) + torch.arange(Sq, device=k_cache.device).view(1, Sq) + bidx = torch.arange(B, device=k_cache.device).view(B, 1).expand(B, Sq) + k_cache[bidx, pos] = k.to(k_cache.dtype) + v_cache[bidx, pos] = v.to(v_cache.dtype) + + +def flash_attn_with_kvcache( + q: Tensor, + k_cache: Tensor, + v_cache: Tensor, + k: Optional[Tensor] = None, + v: Optional[Tensor] = None, + rotary_cos: Optional[Tensor] = None, + rotary_sin: Optional[Tensor] = None, + cache_seqlens=None, + block_table: Optional[Tensor] = None, + softmax_scale: Optional[float] = None, + causal: bool = True, + window_size: Tuple[int, ...] = (-1, -1, 0), + s_aux: Optional[Tensor] = None, + return_lse: bool = False, + **kwargs, +) -> Union[Tensor, Tuple[Tensor, ...]]: + """FlashAttention against a KV cache, built on the CK split-KV kernel. + + ``q`` is ``(batch, seqlen_q, nheads, headdim)``. For a contiguous cache, + ``k_cache``/``v_cache`` are ``(batch, seqlen_cache, nheads_k, headdim)`` and + ``cache_seqlens`` (int or ``(batch,)``) is the number of valid cache entries + *before* appending. If ``k``/``v`` (the new tokens) are given, they are + written into the cache in place. The query attends to the valid prefix + (``cache_seqlens`` + ``seqlen_q``) with bottom-right causal alignment. + + Notes / current limitations: + - Rotary embedding is not applied here; pass ``q``/``k`` already rotated + (``rotary_cos``/``rotary_sin`` are rejected). + - Paged caches are addressed via ``block_table`` (forwarded to the split-KV + varlen kernel); in-cache append for paged layouts must be done by the + caller (pass ``k=v=None``). + """ + if rotary_cos is not None or rotary_sin is not None: + raise NotImplementedError( + "flash_attn_with_kvcache does not apply rotary; rotate q/k beforehand." + ) + if softmax_scale is None: + softmax_scale = q.shape[-1] ** (-0.5) + window_size_left, window_size_right, sink_size = _unpack_window(window_size) + B, Sq, Hq, D = q.shape + s_aux = _prep_sink(s_aux, Hq, q.device) + sink_size = _effective_sink_size(sink_size, s_aux) + + paged = block_table is not None + + # ---- append new tokens into the cache (contiguous layout only) ---- + if k is not None and not paged: + assert cache_seqlens is not None, "cache_seqlens required to append into the cache" + _append_to_contiguous_cache(k_cache, v_cache, k, v, cache_seqlens, Sq) + elif k is not None and paged: + raise NotImplementedError( + "in-cache append for paged layouts is not supported; append externally " + "and call with k=v=None." + ) + + appended = 0 if k is None else Sq + + # ---- key lengths ---- + if cache_seqlens is None: + klens = torch.full((B,), k_cache.shape[1], device=q.device, dtype=torch.long) + elif isinstance(cache_seqlens, int): + klens = torch.full((B,), cache_seqlens + appended, device=q.device, dtype=torch.long) + else: + klens = cache_seqlens.to(q.device, torch.long) + appended + + uniform = bool((klens == klens[0]).all().item()) + + # ---- fast path: contiguous cache, uniform length -> dense kernel ---- + if not paged and uniform: + L = int(klens[0].item()) + return flash_attn_func( + q, k_cache[:, :L], v_cache[:, :L], + softmax_scale=softmax_scale, causal=causal, + window_size=(window_size_left, window_size_right, sink_size), + s_aux=s_aux, return_lse=return_lse, + ) + + # ---- general path: variable cache lengths -> split-KV varlen kernel ---- + cu_q = torch.arange(0, (B + 1) * Sq, Sq, device=q.device, dtype=torch.int32) + cu_k = torch.zeros(B + 1, device=q.device, dtype=torch.int32) + cu_k[1:] = torch.cumsum(klens, 0).to(torch.int32) + q_packed = q.reshape(B * Sq, Hq, D) + if paged: + k_packed, v_packed = k_cache, v_cache # kernel gathers via block_table + else: + k_packed = torch.cat([k_cache[b, : int(klens[b])] for b in range(B)], dim=0) + v_packed = torch.cat([v_cache[b, : int(klens[b])] for b in range(B)], dim=0) + + out = flash_attn_varlen_func( + q_packed, k_packed, v_packed, cu_q, cu_k, Sq, int(klens.max().item()), + softmax_scale=softmax_scale, causal=causal, + window_size=(window_size_left, window_size_right, sink_size), + block_table=block_table, s_aux=s_aux, return_lse=return_lse, + ) + if return_lse: + out, lse = out + return out.reshape(B, Sq, Hq, D), lse + return out.reshape(B, Sq, Hq, D) diff --git a/build/torch210-rocm71-x86_64-linux/symbols.json b/build/torch210-rocm71-x86_64-linux/symbols.json new file mode 100644 index 0000000000000000000000000000000000000000..e8348db4bd94712352e7fb3e83b1d94f47c2814b --- /dev/null +++ b/build/torch210-rocm71-x86_64-linux/symbols.json @@ -0,0 +1,349 @@ +{ + "schema_version": 1, + "module": "_aiter_flash_attn_ck_rocm_fe2bb66", + "functions": [ + { + "name": "flash_attn_func", + "qualname": "flash_attn_func", + "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha", + "kind": "function", + "doc": "Dense FlashAttention forward.\n\n``q, k, v`` have shape ``(batch, seqlen, nheads, headdim)``. ``window_size``\nis ``(left, right, sink_size)``; ``s_aux`` is an optional ``(nheads_q,)``\nfp32 tensor of learnable sink logits. Extra keyword arguments (e.g. those\npassed by the transformers flash-attention interface) are ignored.", + "signature": { + "parameters": [ + { + "name": "q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "v", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "dropout_p", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "0.0", + "annotation": "float" + }, + { + "name": "softmax_scale", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "float | None" + }, + { + "name": "causal", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "window_size", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "(-1, -1, 0)", + "annotation": "Tuple[int, ...]" + }, + { + "name": "bias", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "alibi_slopes", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "return_lse", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "return_attn_probs", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "s_aux", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "kwargs", + "kind": "VAR_KEYWORD", + "default": null, + "annotation": null + } + ], + "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]" + } + }, + { + "name": "flash_attn_varlen_func", + "qualname": "flash_attn_varlen_func", + "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha", + "kind": "function", + "doc": "Variable-length / packed FlashAttention forward.\n\n``q`` has shape ``(total_q, nheads, headdim)``; ``cu_seqlens_*`` are int32\ncumulative offsets of shape ``(batch + 1,)``. Sinks are supported through\n``window_size[2]`` / ``s_aux`` as in :func:`flash_attn_func`. Extra keyword\narguments (e.g. from the transformers flash-attention interface) are ignored.", + "signature": { + "parameters": [ + { + "name": "q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "v", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "cu_seqlens_q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "cu_seqlens_k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "max_seqlen_q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "int" + }, + { + "name": "max_seqlen_k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "int" + }, + { + "name": "dropout_p", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "0.0", + "annotation": "float" + }, + { + "name": "softmax_scale", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "float | None" + }, + { + "name": "causal", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "window_size", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "(-1, -1, 0)", + "annotation": "Tuple[int, ...]" + }, + { + "name": "bias", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "alibi_slopes", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "return_lse", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "return_attn_probs", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "block_table", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "min_seqlen_q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "0", + "annotation": "int" + }, + { + "name": "logits_soft_cap", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "0.0", + "annotation": "float" + }, + { + "name": "zero_tensors", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "cu_seqlens_q_padded", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "cu_seqlens_k_padded", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "s_aux", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "kwargs", + "kind": "VAR_KEYWORD", + "default": null, + "annotation": null + } + ], + "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]" + } + }, + { + "name": "flash_attn_with_kvcache", + "qualname": "flash_attn_with_kvcache", + "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha", + "kind": "function", + "doc": "FlashAttention against a KV cache, built on the CK split-KV kernel.\n\n``q`` is ``(batch, seqlen_q, nheads, headdim)``. For a contiguous cache,\n``k_cache``/``v_cache`` are ``(batch, seqlen_cache, nheads_k, headdim)`` and\n``cache_seqlens`` (int or ``(batch,)``) is the number of valid cache entries\n*before* appending. If ``k``/``v`` (the new tokens) are given, they are\nwritten into the cache in place. The query attends to the valid prefix\n(``cache_seqlens`` + ``seqlen_q``) with bottom-right causal alignment.\n\nNotes / current limitations:\n- Rotary embedding is not applied here; pass ``q``/``k`` already rotated\n (``rotary_cos``/``rotary_sin`` are rejected).\n- Paged caches are addressed via ``block_table`` (forwarded to the split-KV\n varlen kernel); in-cache append for paged layouts must be done by the\n caller (pass ``k=v=None``).", + "signature": { + "parameters": [ + { + "name": "q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "k_cache", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "v_cache", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "v", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "rotary_cos", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "rotary_sin", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "cache_seqlens", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": null + }, + { + "name": "block_table", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "softmax_scale", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "float | None" + }, + { + "name": "causal", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "True", + "annotation": "bool" + }, + { + "name": "window_size", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "(-1, -1, 0)", + "annotation": "Tuple[int, ...]" + }, + { + "name": "s_aux", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "return_lse", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "kwargs", + "kind": "VAR_KEYWORD", + "default": null, + "annotation": null + } + ], + "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]" + } + } + ], + "layers": [] +} diff --git a/build/torch211-rocm71-x86_64-linux/__init__.py b/build/torch211-rocm71-x86_64-linux/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..73acb58c1142a1ab2a6ed9d6f34bc6a53ab5e9b5 --- /dev/null +++ b/build/torch211-rocm71-x86_64-linux/__init__.py @@ -0,0 +1,27 @@ +"""AITER Composable-Kernel FlashAttention kernels for AMD ROCm (compiled HIP). + +Compiled (Composable Kernel / HIP) counterpart to the Triton-based +``aiter-flash-attn`` package, repackaged from the CK FMHA forward path of the +`ROCm/aiter `_ project as a self-contained +Hugging Face Hub kernel. Exposes the FlashAttention forward entry points with +learnable attention sink support (e.g. gpt-oss). +""" + +from .mha import ( + flash_attn_func, + flash_attn_varlen_func, + flash_attn_with_kvcache, +) + + +__kernel_metadata__ = { + "license": "mit", +} + + +__all__ = [ + "__kernel_metadata__", + "flash_attn_func", + "flash_attn_varlen_func", + "flash_attn_with_kvcache", +] diff --git a/build/torch211-rocm71-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so b/build/torch211-rocm71-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so new file mode 100644 index 0000000000000000000000000000000000000000..782791f6d1b44c8d09659126a83a4cb753fcd439 --- /dev/null +++ b/build/torch211-rocm71-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a386f0a7da580b6efa0dd9ffac2d1a4be1382a709f490ab786bf184b7fb4744 +size 204251704 diff --git a/build/torch211-rocm71-x86_64-linux/_ops.py b/build/torch211-rocm71-x86_64-linux/_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..2b4dd56c220b61fd2c16e0c1a02ee91a491cef0c --- /dev/null +++ b/build/torch211-rocm71-x86_64-linux/_ops.py @@ -0,0 +1,9 @@ +import torch +from . import _aiter_flash_attn_ck_rocm_fe2bb66 +ops = torch.ops._aiter_flash_attn_ck_rocm_fe2bb66 + +def add_op_namespace_prefix(op_name: str): + """ + Prefix op by namespace. + """ + return f"_aiter_flash_attn_ck_rocm_fe2bb66::{op_name}" diff --git a/build/torch211-rocm71-x86_64-linux/metadata.json b/build/torch211-rocm71-x86_64-linux/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..a9d2401e54b2cc3c825e7452f77f6deb57803a29 --- /dev/null +++ b/build/torch211-rocm71-x86_64-linux/metadata.json @@ -0,0 +1,37 @@ +{ + "name": "aiter-flash-attn-ck", + "id": "_aiter_flash_attn_ck_rocm_fe2bb66", + "version": 1, + "kernels-minver": "0.14.0", + "license": "MIT", + "python-depends": [], + "kernel-depends": [], + "backend": { + "type": "rocm", + "archs": [ + "gfx942", + "gfx950" + ] + }, + "digest": { + "algorithm": "sha256", + "files": { + "__init__.py": "7wj1GTm2bqTLLCyT4xLuAk/+JWYrhINjY/ODMNWhgZ0=", + "_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so": "ijhvCn2lgLbvoN2f+sLRpL4TgqcJ9JCreGvxhLf7R0Q=", + "_ops.py": "4WkdYDWWJp60zep+oW/5ym9vIeVMNfrgRCkP3o/w7yU=", + "mha.py": "zCAjY4O+DiCzmEL+W9sqXxZaqjsj4RJbWFcyPpUZejk=", + "symbols.json": "rhnF4H8Jsb6I+YPxIVFMgnLcTQJODEXbZ7ge77NZXAU=" + } + }, + "provenance": { + "kernel-builder": { + "version": "0.18.0-dev0", + "commit": "d054906a13710b811b96697b4635f03020842733", + "dirty": false + }, + "kernel": { + "commit": "fe2bb66fee6c2a77eccf08a0508dc57428d488ed", + "dirty": false + } + } +} \ No newline at end of file diff --git a/build/torch211-rocm71-x86_64-linux/metadata.json.sigstore b/build/torch211-rocm71-x86_64-linux/metadata.json.sigstore new file mode 100644 index 0000000000000000000000000000000000000000..3909a048f578c78a011eff6f72a29a725b6bac05 --- /dev/null +++ b/build/torch211-rocm71-x86_64-linux/metadata.json.sigstore @@ -0,0 +1 @@ +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json", "verificationMaterial":{"certificate":{"rawBytes":"MIIHLTCCBrKgAwIBAgIUEmpsGYHCGCL1f77EXMK1mpdslE0wCgYIKoZIzj0EAwMwNzEVMBMGA1UEChMMc2lnc3RvcmUuZGV2MR4wHAYDVQQDExVzaWdzdG9yZS1pbnRlcm1lZGlhdGUwHhcNMjYxMDE5MjIzMTEzWhcNMjYxMDE5MjI0MTEzWjAAMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE4Fy9kpEGzuCqkOEdVCP+RLOaIvA6RK5RuqsBRqXOQ1oE/UwcaBM8VLmKzmbspKP8LZxu8R+MYRHT7vmO2qJGLKOCBdEwggXNMA4GA1UdDwEB/wQEAwIHgDATBgNVHSUEDDAKBggrBgEFBQcDAzAfBgNVHSMEGDAWgBTf0+nPViQRlvmo2OkoVaLGLhhkPzBrBgNVHREBAf8EYTBfhl1odHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkvLmdpdGh1Yi93b3JrZmxvd3MvYnVpbGQueWFtbEByZWZzL2hlYWRzL21haW4wOQYKKwYBBAGDvzABAQQraHR0cHM6Ly90b2tlbi5hY3Rpb25zLmdpdGh1YnVzZXJjb250ZW50LmNvbTAfBgorBgEEAYO/MAECBBF3b3JrZmxvd19kaXNwYXRjaDA2BgorBgEEAYO/MAEDBChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMBMGCisGAQQBg78wAQQEBUJ1aWxkMCsGCisGAQQBg78wAQUEHWh1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5MB0GCisGAQQBg78wAQYED3JlZnMvaGVhZHMvbWFpbjA7BgorBgEEAYO/MAEIBC0MK2h0dHBzOi8vdG9rZW4uYWN0aW9ucy5naXRodWJ1c2VyY29udGVudC5jb20wbQYKKwYBBAGDvzABCQRfDF1odHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkvLmdpdGh1Yi93b3JrZmxvd3MvYnVpbGQueWFtbEByZWZzL2hlYWRzL21haW4wOAYKKwYBBAGDvzABCgQqDChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMBsGCisGAQQBg78wAQsEDQwLc2VsZi1ob3N0ZWQwQAYKKwYBBAGDvzABDAQyDDBodHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkwOAYKKwYBBAGDvzABDQQqDChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMB8GCisGAQQBg78wAQ4EEQwPcmVmcy9oZWFkcy9tYWluMBoGCisGAQQBg78wAQ8EDAwKMTA3MTQ3NTUyOTAuBgorBgEEAYO/MAEQBCAMHmh0dHBzOi8vZ2l0aHViLmNvbS9odWdnaW5nZmFjZTAYBgorBgEEAYO/MAERBAoMCDI1NzIwNzQzMG0GCisGAQQBg78wARIEXwxdaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5Ly5naXRodWIvd29ya2Zsb3dzL2J1aWxkLnlhbWxAcmVmcy9oZWFkcy9tYWluMDgGCisGAQQBg78wARMEKgwoZmUyYmI2NmZlZTZjMmE3N2VjY2YwOGEwNTA4ZGM1NzQyOGQ0ODhlZDAhBgorBgEEAYO/MAEUBBMMEXdvcmtmbG93X2Rpc3BhdGNoMGQGCisGAQQBg78wARUEVgxUaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5L2FjdGlvbnMvcnVucy8zOTIzOTUxODEwNC9hdHRlbXB0cy8xMBYGCisGAQQBg78wARYECAwGcHVibGljMEYGCisGAQQBg78wARgEOAw2cmVwbzpodWdnaW5nZmFjZS9rZXJuZWxzLWNvbW11bml0eTpyZWY6cmVmcy9oZWFkcy9tYWluMIGKBgorBgEEAdZ5AgQCBHwEegB4AHYA3T0wasbHETJjGR4cmWc3AqJKXrjePK3/h4pygC8p7o4AAAGhVkpQHwAABAMARzBFAiEA5+eh0gx7+UP+mTMZ7dYGtXAGoMr/u3B/1ATobugZirgCIDMCumLOPPvo9Ns0vC8ybrQt7fXH5h98acJfF9aYV1fuMAoGCCqGSM49BAMDA2kAMGYCMQC1gggEuYtV0xjSBQFLnBzb2NdSMXXnrwNQMbOsqeldlpTGoUouHMgT0NboOvp74NcCMQCy79wgE2YbxXDZW7ye5v9a8T72ozdyyAG3+bFixIHj0rx86inTdkuqjr8I90iw9ok="}, "tlogEntries":[{"logIndex":"3327211453", "logId":{"keyId":"wNI9atQGlz+VWfO6LRygH4QUfY/8W4RFwiT5i5WRgB0="}, "kindVersion":{"kind":"hashedrekord", "version":"0.0.1"}, "integratedTime":"1792449073", "inclusionPromise":{"signedEntryTimestamp":"MEQCICqb2FveJcObQ1buwFT+0NvR4ZCa0apWIyoOzyrWsgzYAiB4YVmCelHrY9h3rlc+qWSF+O+hNiQ/syPijjNsHJwahg=="}, "inclusionProof":{"logIndex":"3205307191", "rootHash":"iiOp2twenBLzbfls9348G+Zypx6NFcfs3+o2fG/C/fk=", "treeSize":"3205307194", "hashes":["fquilGZratvzTDiv84GzP8s55UYAN0iUI2kGqK20cXE=", "fugeqQEriw+je6JcoVNkTkv+UD+IzIQtDv/zaj8Quj8=", "4hRzU/5ELB96P1nZkhgwpYYVgFAr1DXDztt20pO7If4=", "rM02OjNrzut1+wWBYgEkyMNTWk3J8j72VYpb5yxWm/g=", "gIcTbwmbW4Jinimt4uWlbE/g8i6p+zi5IXTWev0jxFY=", "iekAzKnn2vj8KKbMbX8Obrd5/3Mxh3/6lsToOqKeuTU=", "AJlvaM7WQKZg0FAjSdh7RzJXyGKaOW5DVz46FbaUPZE=", "t4Yy4kaoiJ3+VosqXw5Ogmg+A7p/E6y8+JhqWbalMbI=", "5egUlZ4Pi0Ix/qLyCDXocsWywJCHqrde/S06vOZvlOM=", "QsXufUpoKtSjKAB19K1G00umQpWvPHlBNuRGFiiVjQQ=", "QIMkwQbwAewhLJd//5FFwI3SnyNQBGjCgJrelfG8vhc=", "Et17CACUv9QN64irUJuBjsrgRGkGnnJ9xacRV/aBdwQ=", "lqgEwkxhDMs/gdIAE4lmjfMwg7nDmv7lda6O0LGliCw=", "HovTPBWzoErvUg1u5sIi3x1ZonZ7yJekVsBrmpomCIA=", "G+nk7mgE5h6dUmv2UY7NDJrY3WvrZY31raaO1AETt+o=", "qlTWNDzERnTBQVcHHXXVPjiUP0SsikGxfcxVlgqVFDo=", "gg9GNApNke3+zNhl4tsWtmIycCFrOMCB3lys6XuSQNQ=", "NlOclldmKPA1tMKIskqHAwuQDYHp31YwpK+OOMrMccc=", "qxzHanAzz57SDdmJe0B7bJK72NTIBbwEMGBBKvDOROw=", "xH/DCseLHr9eKoYT8qsORZK7zVdEGYWHuVtsVrD95wY="], "checkpoint":{"envelope":"rekor.sigstore.dev - 1193050959916656506\n3205307194\niiOp2twenBLzbfls9348G+Zypx6NFcfs3+o2fG/C/fk=\n\n— rekor.sigstore.dev wNI9ajBGAiEA4MrXZ0/pNdLZFmSAyN8SsGiWgHq/FNXsTxStKfSyb5YCIQD4kmm+enQutJHTg7MkOS+ckvQ2VGJuDl8kjANqXlKi6Q==\n"}}, "canonicalizedBody":"eyJhcGlWZXJzaW9uIjoiMC4wLjEiLCJraW5kIjoiaGFzaGVkcmVrb3JkIiwic3BlYyI6eyJkYXRhIjp7Imhhc2giOnsiYWxnb3JpdGhtIjoic2hhMjU2IiwidmFsdWUiOiI1MGE4MDNkMjI4NDhkOGExYThmNmRhYWRmNWJkZTVhYWQ2ZDk3MzUwNjI1OTIwY2E4OTVmYzAyNDM2ZjVkNDQ1In19LCJzaWduYXR1cmUiOnsiY29udGVudCI6Ik1FUUNJQm8rZ3Ftb09qeUcxNVFpUkRRd3d4QTNuMVExVi83dUJrOFRLYlZ5ZEpkOEFpQnFNcC96SlZkditjN1IzSHM2UlFnbE5EQmFhZnJYNzRUZ25TQzJ0dDdTTEE9PSIsInB1YmxpY0tleSI6eyJjb250ZW50IjoiTFMwdExTMUNSVWRKVGlCRFJWSlVTVVpKUTBGVVJTMHRMUzB0Q2sxSlNVaE1WRU5EUW5KTFowRjNTVUpCWjBsVlJXMXdjMGRaU0VOSFEwd3haamMzUlZoTlN6RnRjR1J6YkVVd2QwTm5XVWxMYjFwSmVtb3dSVUYzVFhjS1RucEZWazFDVFVkQk1WVkZRMmhOVFdNeWJHNWpNMUoyWTIxVmRWcEhWakpOVWpSM1NFRlpSRlpSVVVSRmVGWjZZVmRrZW1SSE9YbGFVekZ3WW01U2JBcGpiVEZzV2tkc2FHUkhWWGRJYUdOT1RXcFplRTFFUlRWTmFrbDZUVlJGZWxkb1kwNU5hbGw0VFVSRk5VMXFTVEJOVkVWNlYycEJRVTFHYTNkRmQxbElDa3R2V2tsNmFqQkRRVkZaU1V0dldrbDZhakJFUVZGalJGRm5RVVUwUm5rNWEzQkZSM3AxUTNGclQwVmtWa05RSzFKTVQyRkpka0UyVWtzMVVuVnhjMElLVW5GWVQxRXhiMFV2VlhkallVSk5PRlpNYlV0NmJXSnpjRXRRT0V4YWVIVTRVaXROV1ZKSVZEZDJiVTh5Y1VwSFRFdFBRMEprUlhkbloxaE9UVUUwUndwQk1WVmtSSGRGUWk5M1VVVkJkMGxJWjBSQlZFSm5UbFpJVTFWRlJFUkJTMEpuWjNKQ1owVkdRbEZqUkVGNlFXWkNaMDVXU0ZOTlJVZEVRVmRuUWxSbUNqQXJibEJXYVZGU2JIWnRiekpQYTI5V1lVeEhUR2hvYTFCNlFuSkNaMDVXU0ZKRlFrRm1PRVZaVkVKbWFHd3hiMlJJVW5kamVtOTJUREprY0dSSGFERUtXV2sxYW1JeU1IWmhTRlp1V2pKc2RWb3lXbWhaTWxWMllUSldlV0p0Vm5OamVURnFZakl4ZEdSWE5YQmtTR3QyVEcxa2NHUkhhREZaYVRrellqTktjZ3BhYlhoMlpETk5kbGx1Vm5CaVIxRjFaVmRHZEdKRlFubGFWMXA2VERKb2JGbFhVbnBNTWpGb1lWYzBkMDlSV1V0TGQxbENRa0ZIUkhaNlFVSkJVVkZ5Q21GSVVqQmpTRTAyVEhrNU1HSXlkR3hpYVRWb1dUTlNjR0l5TlhwTWJXUndaRWRvTVZsdVZucGFXRXBxWWpJMU1GcFhOVEJNYlU1MllsUkJaa0puYjNJS1FtZEZSVUZaVHk5TlFVVkRRa0pHTTJJelNuSmFiWGgyWkRFNWEyRllUbmRaV0ZKcVlVUkJNa0puYjNKQ1owVkZRVmxQTDAxQlJVUkNRMmh0V2xSS2FRcFphbGt5V20xV2JFNXRUWGxaVkdNeldsZE9hbHBxUVRSWlZFRXhUVVJvYTFsNlZUTk9SRWswV2tSUk5FOUhWbXROUWsxSFEybHpSMEZSVVVKbk56aDNDa0ZSVVVWQ1ZVb3hZVmQ0YTAxRGMwZERhWE5IUVZGUlFtYzNPSGRCVVZWRlNGZG9NVm95WkhCaWJXUnRXVmRPYkV3eWRHeGpiVFZzWWtoTmRGa3lPWFFLWWxoV2RXRllValZOUWpCSFEybHpSMEZSVVVKbk56aDNRVkZaUlVRelNteGFiazEyWVVkV2FGcElUWFppVjBad1ltcEJOMEpuYjNKQ1owVkZRVmxQTHdwTlFVVkpRa013VFVzeWFEQmtTRUo2VDJrNGRtUkhPWEphVnpSMVdWZE9NR0ZYT1hWamVUVnVZVmhTYjJSWFNqRmpNbFo1V1RJNWRXUkhWblZrUXpWcUNtSXlNSGRpVVZsTFMzZFpRa0pCUjBSMmVrRkNRMUZTWmtSR01XOWtTRkozWTNwdmRrd3laSEJrUjJneFdXazFhbUl5TUhaaFNGWnVXakpzZFZveVdtZ0tXVEpWZG1FeVZubGliVlp6WTNreGFtSXlNWFJrVnpWd1pFaHJka3h0WkhCa1IyZ3hXV2s1TTJJelNuSmFiWGgyWkROTmRsbHVWbkJpUjFGMVpWZEdkQXBpUlVKNVdsZGFla3d5YUd4WlYxSjZUREl4YUdGWE5IZFBRVmxMUzNkWlFrSkJSMFIyZWtGQ1EyZFJjVVJEYUcxYVZFcHBXV3BaTWxwdFZteE9iVTE1Q2xsVVl6TmFWMDVxV21wQk5GbFVRVEZOUkdocldYcFZNMDVFU1RSYVJGRTBUMGRXYTAxQ2MwZERhWE5IUVZGUlFtYzNPSGRCVVhORlJGRjNUR015Vm5NS1dta3hiMkl6VGpCYVYxRjNVVUZaUzB0M1dVSkNRVWRFZG5wQlFrUkJVWGxFUkVKdlpFaFNkMk42YjNaTU1tUndaRWRvTVZscE5XcGlNakIyWVVoV2JncGFNbXgxV2pKYWFGa3lWWFpoTWxaNVltMVdjMk41TVdwaU1qRjBaRmMxY0dSSWEzZFBRVmxMUzNkWlFrSkJSMFIyZWtGQ1JGRlJjVVJEYUcxYVZFcHBDbGxxV1RKYWJWWnNUbTFOZVZsVVl6TmFWMDVxV21wQk5GbFVRVEZOUkdocldYcFZNMDVFU1RSYVJGRTBUMGRXYTAxQ09FZERhWE5IUVZGUlFtYzNPSGNLUVZFMFJVVlJkMUJqYlZadFkzazViMXBYUm10amVUbDBXVmRzZFUxQ2IwZERhWE5IUVZGUlFtYzNPSGRCVVRoRlJFRjNTMDFVUVROTlZGRXpUbFJWZVFwUFZFRjFRbWR2Y2tKblJVVkJXVTh2VFVGRlVVSkRRVTFJYldnd1pFaENlazlwT0haYU1td3dZVWhXYVV4dFRuWmlVemx2WkZka2JtRlhOVzVhYlVacUNscFVRVmxDWjI5eVFtZEZSVUZaVHk5TlFVVlNRa0Z2VFVORVNURk9la2wzVG5wUmVrMUhNRWREYVhOSFFWRlJRbWMzT0hkQlVrbEZXSGQ0WkdGSVVqQUtZMGhOTmt4NU9XNWhXRkp2WkZkSmRWa3lPWFJNTW1neFdqSmtjR0p0WkcxWlYwNXNUREowYkdOdE5XeGlTRTEwV1RJNWRHSllWblZoV0ZJMVRIazFiZ3BoV0ZKdlpGZEpkbVF5T1hsaE1scHpZak5rZWt3eVNqRmhWM2hyVEc1c2FHSlhlRUZqYlZadFkzazViMXBYUm10amVUbDBXVmRzZFUxRVowZERhWE5IQ2tGUlVVSm5OemgzUVZKTlJVdG5kMjlhYlZWNVdXMUpNazV0V214YVZGcHFUVzFGTTA0eVZtcFpNbGwzVDBkRmQwNVVRVFJhUjAweFRucFJlVTlIVVRBS1QwUm9iRnBFUVdoQ1oyOXlRbWRGUlVGWlR5OU5RVVZWUWtKTlRVVllaSFpqYlhSdFlrYzVNMWd5VW5Cak0wSm9aRWRPYjAxSFVVZERhWE5IUVZGUlFncG5OemgzUVZKVlJWWm5lRlZoU0ZJd1kwaE5Oa3g1T1c1aFdGSnZaRmRKZFZreU9YUk1NbWd4V2pKa2NHSnRaRzFaVjA1c1RESjBiR050Tld4aVNFMTBDbGt5T1hSaVdGWjFZVmhTTlV3eVJtcGtSMngyWW01TmRtTnVWblZqZVRoNlQxUkplazlVVlhoUFJFVjNUa001YUdSSVVteGlXRUl3WTNrNGVFMUNXVWNLUTJselIwRlJVVUpuTnpoM1FWSlpSVU5CZDBkalNGWnBZa2RzYWsxRldVZERhWE5IUVZGUlFtYzNPSGRCVW1kRlQwRjNNbU50Vm5kaWVuQnZaRmRrYmdwaFZ6VnVXbTFHYWxwVE9YSmFXRXAxV2xkNGVreFhUblppVnpFeFltMXNNR1ZVY0hsYVYxazJZMjFXYldONU9XOWFWMFpyWTNrNWRGbFhiSFZOU1VkTENrSm5iM0pDWjBWRlFXUmFOVUZuVVVOQ1NIZEZaV2RDTkVGSVdVRXpWREIzWVhOaVNFVlVTbXBIVWpSamJWZGpNMEZ4U2t0WWNtcGxVRXN6TDJnMGNIa0taME00Y0Rkdk5FRkJRVWRvVm10d1VVaDNRVUZDUVUxQlVucENSa0ZwUlVFMUsyVm9NR2Q0Tnl0VlVDdHRWRTFhTjJSWlIzUllRVWR2VFhJdmRUTkNMd294UVZSdlluVm5XbWx5WjBOSlJFMURkVzFNVDFCUWRtODVUbk13ZGtNNGVXSnlVWFEzWmxoSU5XZzVPR0ZqU21aR09XRlpWakZtZFUxQmIwZERRM0ZIQ2xOTk5EbENRVTFFUVRKclFVMUhXVU5OVVVNeFoyZG5SWFZaZEZZd2VHcFRRbEZHVEc1Q2VtSXlUbVJUVFZoWWJuSjNUbEZOWWs5emNXVnNaR3h3VkVjS2IxVnZkVWhOWjFRd1RtSnZUM1p3TnpST1kwTk5VVU41TnpsM1owVXlXV0o0V0VSYVZ6ZDVaVFYyT1dFNFZEY3liM3BrZVhsQlJ6TXJZa1pwZUVsSWFnb3djbmc0Tm1sdVZHUnJkWEZxY2poSk9UQnBkemx2YXowS0xTMHRMUzFGVGtRZ1EwVlNWRWxHU1VOQlZFVXRMUzB0TFFvPSJ9fX19"}], "timestampVerificationData":{"rfc3161Timestamps":[{"signedTimestamp":"MIICyjADAgEAMIICwQYJKoZIhvcNAQcCoIICsjCCAq4CAQMxDTALBglghkgBZQMEAgEwgbgGCyqGSIb3DQEJEAEEoIGoBIGlMIGiAgEBBgkrBgEEAYO/MAIwMTANBglghkgBZQMEAgEFAAQgve8SsvliP+kBGAv5uenHP7txgoqMJQ//kJyYCG+oIZQCFQCFqztCSeTNbThwd1EK8pfaDGvUzxgPMjAyNjEwMTkyMjMxMTNaMAMCAQGgMqQwMC4xFTATBgNVBAoTDHNpZ3N0b3JlLmRldjEVMBMGA1UEAxMMc2lnc3RvcmUtdHNhoAAxggHbMIIB1wIBATBRMDkxFTATBgNVBAoTDHNpZ3N0b3JlLmRldjEgMB4GA1UEAxMXc2lnc3RvcmUtdHNhLXNlbGZzaWduZWQCFDoTVC8MkGHuvMFDL8uKjosqI4sMMAsGCWCGSAFlAwQCAaCB/DAaBgkqhkiG9w0BCQMxDQYLKoZIhvcNAQkQAQQwHAYJKoZIhvcNAQkFMQ8XDTI2MTAxOTIyMzExM1owLwYJKoZIhvcNAQkEMSIEIABz+BxWlSmUQSkAqKq3rF9he+spJMqg2Knzgd2nAeimMIGOBgsqhkiG9w0BCRACLzF/MH0wezB5BCCF+Se8B6tiysO0Q1bBDvyBssaIP9p6uebYcNnROs0FtzBVMD2kOzA5MRUwEwYDVQQKEwxzaWdzdG9yZS5kZXYxIDAeBgNVBAMTF3NpZ3N0b3JlLXRzYS1zZWxmc2lnbmVkAhQ6E1QvDJBh7rzBQy/Lio6LKiOLDDAKBggqhkjOPQQDAgRnMGUCMDCtCa0AeNpbVDBgccxCXaS9qeyqeakv/2BZZVHiqKiCNrXU79GQ6q9FTWcg6JIxMgIxANVyI0gY8+W9UpPT0WZEBk0Fo2APA2MAPz5gH/ksPyYJIeHDeKqZjnLfkEacqtMAYw=="}]}}, "messageSignature":{"messageDigest":{"algorithm":"SHA2_256", "digest":"UKgD0ihI2KGo9tqt9b3lqtbZc1BiWSDKiV/AJDb11EU="}, "signature":"MEQCIBo+gqmoOjyG15QiRDQwwxA3n1Q1V/7uBk8TKbVydJd8AiBqMp/zJVdv+c7R3Hs6RQglNDBaafrX74TgnSC2tt7SLA=="}} \ No newline at end of file diff --git a/build/torch211-rocm71-x86_64-linux/mha.py b/build/torch211-rocm71-x86_64-linux/mha.py new file mode 100644 index 0000000000000000000000000000000000000000..01e2d6a0fbee98d27129f2eb695ac38df73e6487 --- /dev/null +++ b/build/torch211-rocm71-x86_64-linux/mha.py @@ -0,0 +1,332 @@ +"""FlashAttention forward entry points backed by aiter's Composable-Kernel FMHA. + +This is the compiled (HIP / Composable Kernel) counterpart to the Triton +``aiter-flash-attn`` package. Only the *forward* pass is provided (dense and +variable-length), including support for learnable attention sinks via +``window_size[2]`` / ``s_aux`` (e.g. gpt-oss). The op set ships exclusively +the CK ``mha_fwd`` / ``mha_varlen_fwd`` kernels, so the sink path is always +taken (no fall-through to a non-sink ASM kernel). +""" + +from typing import Optional, Tuple, Union + +import torch +from torch import Tensor + +from ._ops import ops + + +def _round_multiple(x: int, m: int) -> int: + return (x + m - 1) // m * m + + +def _maybe_pad_headdim(t: Tensor, target: int) -> Tensor: + og = t.size(-1) + if og == target: + return t + return torch.nn.functional.pad(t, [0, target - og]) + + +def _unpack_window(window_size) -> Tuple[int, int, int]: + left = int(window_size[0]) + right = int(window_size[1]) + sink = int(window_size[2]) if len(window_size) == 3 else 0 + return left, right, sink + + +def _prep_sink(s_aux: Optional[Tensor], nheads: int, device) -> Optional[Tensor]: + if s_aux is None: + return None + assert s_aux.device == device, "s_aux must be on the same device as q" + assert s_aux.shape[0] == nheads, "s_aux must have shape [nheads_q]" + if s_aux.dtype != torch.float32: + s_aux = s_aux.to(torch.float32) + return s_aux + + +def _effective_sink_size(sink_size: int, s_aux: Optional[Tensor]) -> int: + # The CK kernel only takes the sink path when sink_size > 0. Callers using + # the transformers attention interface pass the per-head sink logits as + # ``s_aux`` but leave ``window_size`` a 2-tuple (sink_size == 0), so enable + # a single learnable sink automatically when s_aux is given. + if s_aux is not None and sink_size == 0: + return 1 + return sink_size + + +def flash_attn_func( + q: Tensor, + k: Tensor, + v: Tensor, + dropout_p: float = 0.0, + softmax_scale: Optional[float] = None, + causal: bool = False, + window_size: Tuple[int, ...] = (-1, -1, 0), + bias: Optional[Tensor] = None, + alibi_slopes: Optional[Tensor] = None, + return_lse: bool = False, + return_attn_probs: bool = False, + s_aux: Optional[Tensor] = None, + **kwargs, +) -> Union[Tensor, Tuple[Tensor, ...]]: + """Dense FlashAttention forward. + + ``q, k, v`` have shape ``(batch, seqlen, nheads, headdim)``. ``window_size`` + is ``(left, right, sink_size)``; ``s_aux`` is an optional ``(nheads_q,)`` + fp32 tensor of learnable sink logits. Extra keyword arguments (e.g. those + passed by the transformers flash-attention interface) are ignored. + """ + if softmax_scale is None: + softmax_scale = q.shape[-1] ** (-0.5) + window_size_left, window_size_right, sink_size = _unpack_window(window_size) + + nheads_q = q.size(2) + s_aux = _prep_sink(s_aux, nheads_q, q.device) + sink_size = _effective_sink_size(sink_size, s_aux) + + head_q_og = q.size(-1) + head_v_og = v.size(-1) + head_q = _round_multiple(head_q_og, 8) + head_v = _round_multiple(head_v_og, 8) + q = _maybe_pad_headdim(q, head_q) + k = _maybe_pad_headdim(k, head_q) + v = _maybe_pad_headdim(v, head_v) + + return_softmax = return_attn_probs and dropout_p > 0 + out, softmax_lse, S_dmask, _ = ops.mha_fwd( + q, + k, + v, + dropout_p, + softmax_scale, + causal, + window_size_left, + window_size_right, + sink_size, + return_lse or return_attn_probs, + return_softmax, + None, # cu_seqlens_q + None, # cu_seqlens_kv + None, # out + bias, + alibi_slopes, + None, # q_descale + None, # k_descale + None, # v_descale + s_aux, + None, # gen + ) + + out = out[..., :head_v_og] + if not (return_lse or return_attn_probs): + return out + result = (out,) + if return_lse: + result = result + (softmax_lse,) + if return_attn_probs: + result = result + (S_dmask,) + return result + + +def flash_attn_varlen_func( + q: Tensor, + k: Tensor, + v: Tensor, + cu_seqlens_q: Tensor, + cu_seqlens_k: Tensor, + max_seqlen_q: int, + max_seqlen_k: int, + dropout_p: float = 0.0, + softmax_scale: Optional[float] = None, + causal: bool = False, + window_size: Tuple[int, ...] = (-1, -1, 0), + bias: Optional[Tensor] = None, + alibi_slopes: Optional[Tensor] = None, + return_lse: bool = False, + return_attn_probs: bool = False, + block_table: Optional[Tensor] = None, + min_seqlen_q: int = 0, + logits_soft_cap: float = 0.0, + zero_tensors: bool = False, + cu_seqlens_q_padded: Optional[Tensor] = None, + cu_seqlens_k_padded: Optional[Tensor] = None, + s_aux: Optional[Tensor] = None, + **kwargs, +) -> Union[Tensor, Tuple[Tensor, ...]]: + """Variable-length / packed FlashAttention forward. + + ``q`` has shape ``(total_q, nheads, headdim)``; ``cu_seqlens_*`` are int32 + cumulative offsets of shape ``(batch + 1,)``. Sinks are supported through + ``window_size[2]`` / ``s_aux`` as in :func:`flash_attn_func`. Extra keyword + arguments (e.g. from the transformers flash-attention interface) are ignored. + """ + if softmax_scale is None: + softmax_scale = q.shape[-1] ** (-0.5) + window_size_left, window_size_right, sink_size = _unpack_window(window_size) + + nheads_q = q.size(1) + s_aux = _prep_sink(s_aux, nheads_q, q.device) + sink_size = _effective_sink_size(sink_size, s_aux) + + head_q_og = q.size(-1) + head_v_og = v.size(-1) + head_q = _round_multiple(head_q_og, 8) + head_v = _round_multiple(head_v_og, 8) + q = _maybe_pad_headdim(q, head_q) + k = _maybe_pad_headdim(k, head_q) + v = _maybe_pad_headdim(v, head_v) + + return_softmax = return_attn_probs and dropout_p > 0 + out, softmax_lse, S_dmask, _ = ops.mha_varlen_fwd( + q, + k, + v, + cu_seqlens_q, + cu_seqlens_k, + max_seqlen_q, + max_seqlen_k, + min_seqlen_q, + dropout_p, + softmax_scale, + logits_soft_cap, + zero_tensors, + causal, + window_size_left, + window_size_right, + sink_size, + return_lse or return_attn_probs, + return_softmax, + None, # out + block_table, + bias, + alibi_slopes, + None, # q_descale + None, # k_descale + None, # v_descale + None, # gen + cu_seqlens_q_padded, + cu_seqlens_k_padded, + s_aux, + ) + + out = out[..., :head_v_og] + if not (return_lse or return_attn_probs): + return out + result = (out,) + if return_lse: + result = result + (softmax_lse,) + if return_attn_probs: + result = result + (S_dmask,) + return result + + +def _append_to_contiguous_cache(k_cache, v_cache, k, v, cache_seqlens, Sq): + """In-place append of new k/v into a contiguous cache at per-batch offsets.""" + B = k_cache.shape[0] + if isinstance(cache_seqlens, int): + cache_seqlens = torch.full((B,), cache_seqlens, device=k_cache.device, dtype=torch.long) + pos = cache_seqlens.view(B, 1).to(torch.long) + torch.arange(Sq, device=k_cache.device).view(1, Sq) + bidx = torch.arange(B, device=k_cache.device).view(B, 1).expand(B, Sq) + k_cache[bidx, pos] = k.to(k_cache.dtype) + v_cache[bidx, pos] = v.to(v_cache.dtype) + + +def flash_attn_with_kvcache( + q: Tensor, + k_cache: Tensor, + v_cache: Tensor, + k: Optional[Tensor] = None, + v: Optional[Tensor] = None, + rotary_cos: Optional[Tensor] = None, + rotary_sin: Optional[Tensor] = None, + cache_seqlens=None, + block_table: Optional[Tensor] = None, + softmax_scale: Optional[float] = None, + causal: bool = True, + window_size: Tuple[int, ...] = (-1, -1, 0), + s_aux: Optional[Tensor] = None, + return_lse: bool = False, + **kwargs, +) -> Union[Tensor, Tuple[Tensor, ...]]: + """FlashAttention against a KV cache, built on the CK split-KV kernel. + + ``q`` is ``(batch, seqlen_q, nheads, headdim)``. For a contiguous cache, + ``k_cache``/``v_cache`` are ``(batch, seqlen_cache, nheads_k, headdim)`` and + ``cache_seqlens`` (int or ``(batch,)``) is the number of valid cache entries + *before* appending. If ``k``/``v`` (the new tokens) are given, they are + written into the cache in place. The query attends to the valid prefix + (``cache_seqlens`` + ``seqlen_q``) with bottom-right causal alignment. + + Notes / current limitations: + - Rotary embedding is not applied here; pass ``q``/``k`` already rotated + (``rotary_cos``/``rotary_sin`` are rejected). + - Paged caches are addressed via ``block_table`` (forwarded to the split-KV + varlen kernel); in-cache append for paged layouts must be done by the + caller (pass ``k=v=None``). + """ + if rotary_cos is not None or rotary_sin is not None: + raise NotImplementedError( + "flash_attn_with_kvcache does not apply rotary; rotate q/k beforehand." + ) + if softmax_scale is None: + softmax_scale = q.shape[-1] ** (-0.5) + window_size_left, window_size_right, sink_size = _unpack_window(window_size) + B, Sq, Hq, D = q.shape + s_aux = _prep_sink(s_aux, Hq, q.device) + sink_size = _effective_sink_size(sink_size, s_aux) + + paged = block_table is not None + + # ---- append new tokens into the cache (contiguous layout only) ---- + if k is not None and not paged: + assert cache_seqlens is not None, "cache_seqlens required to append into the cache" + _append_to_contiguous_cache(k_cache, v_cache, k, v, cache_seqlens, Sq) + elif k is not None and paged: + raise NotImplementedError( + "in-cache append for paged layouts is not supported; append externally " + "and call with k=v=None." + ) + + appended = 0 if k is None else Sq + + # ---- key lengths ---- + if cache_seqlens is None: + klens = torch.full((B,), k_cache.shape[1], device=q.device, dtype=torch.long) + elif isinstance(cache_seqlens, int): + klens = torch.full((B,), cache_seqlens + appended, device=q.device, dtype=torch.long) + else: + klens = cache_seqlens.to(q.device, torch.long) + appended + + uniform = bool((klens == klens[0]).all().item()) + + # ---- fast path: contiguous cache, uniform length -> dense kernel ---- + if not paged and uniform: + L = int(klens[0].item()) + return flash_attn_func( + q, k_cache[:, :L], v_cache[:, :L], + softmax_scale=softmax_scale, causal=causal, + window_size=(window_size_left, window_size_right, sink_size), + s_aux=s_aux, return_lse=return_lse, + ) + + # ---- general path: variable cache lengths -> split-KV varlen kernel ---- + cu_q = torch.arange(0, (B + 1) * Sq, Sq, device=q.device, dtype=torch.int32) + cu_k = torch.zeros(B + 1, device=q.device, dtype=torch.int32) + cu_k[1:] = torch.cumsum(klens, 0).to(torch.int32) + q_packed = q.reshape(B * Sq, Hq, D) + if paged: + k_packed, v_packed = k_cache, v_cache # kernel gathers via block_table + else: + k_packed = torch.cat([k_cache[b, : int(klens[b])] for b in range(B)], dim=0) + v_packed = torch.cat([v_cache[b, : int(klens[b])] for b in range(B)], dim=0) + + out = flash_attn_varlen_func( + q_packed, k_packed, v_packed, cu_q, cu_k, Sq, int(klens.max().item()), + softmax_scale=softmax_scale, causal=causal, + window_size=(window_size_left, window_size_right, sink_size), + block_table=block_table, s_aux=s_aux, return_lse=return_lse, + ) + if return_lse: + out, lse = out + return out.reshape(B, Sq, Hq, D), lse + return out.reshape(B, Sq, Hq, D) diff --git a/build/torch211-rocm71-x86_64-linux/symbols.json b/build/torch211-rocm71-x86_64-linux/symbols.json new file mode 100644 index 0000000000000000000000000000000000000000..e8348db4bd94712352e7fb3e83b1d94f47c2814b --- /dev/null +++ b/build/torch211-rocm71-x86_64-linux/symbols.json @@ -0,0 +1,349 @@ +{ + "schema_version": 1, + "module": "_aiter_flash_attn_ck_rocm_fe2bb66", + "functions": [ + { + "name": "flash_attn_func", + "qualname": "flash_attn_func", + "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha", + "kind": "function", + "doc": "Dense FlashAttention forward.\n\n``q, k, v`` have shape ``(batch, seqlen, nheads, headdim)``. ``window_size``\nis ``(left, right, sink_size)``; ``s_aux`` is an optional ``(nheads_q,)``\nfp32 tensor of learnable sink logits. Extra keyword arguments (e.g. those\npassed by the transformers flash-attention interface) are ignored.", + "signature": { + "parameters": [ + { + "name": "q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "v", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "dropout_p", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "0.0", + "annotation": "float" + }, + { + "name": "softmax_scale", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "float | None" + }, + { + "name": "causal", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "window_size", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "(-1, -1, 0)", + "annotation": "Tuple[int, ...]" + }, + { + "name": "bias", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "alibi_slopes", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "return_lse", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "return_attn_probs", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "s_aux", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "kwargs", + "kind": "VAR_KEYWORD", + "default": null, + "annotation": null + } + ], + "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]" + } + }, + { + "name": "flash_attn_varlen_func", + "qualname": "flash_attn_varlen_func", + "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha", + "kind": "function", + "doc": "Variable-length / packed FlashAttention forward.\n\n``q`` has shape ``(total_q, nheads, headdim)``; ``cu_seqlens_*`` are int32\ncumulative offsets of shape ``(batch + 1,)``. Sinks are supported through\n``window_size[2]`` / ``s_aux`` as in :func:`flash_attn_func`. Extra keyword\narguments (e.g. from the transformers flash-attention interface) are ignored.", + "signature": { + "parameters": [ + { + "name": "q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "v", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "cu_seqlens_q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "cu_seqlens_k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "max_seqlen_q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "int" + }, + { + "name": "max_seqlen_k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "int" + }, + { + "name": "dropout_p", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "0.0", + "annotation": "float" + }, + { + "name": "softmax_scale", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "float | None" + }, + { + "name": "causal", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "window_size", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "(-1, -1, 0)", + "annotation": "Tuple[int, ...]" + }, + { + "name": "bias", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "alibi_slopes", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "return_lse", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "return_attn_probs", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "block_table", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "min_seqlen_q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "0", + "annotation": "int" + }, + { + "name": "logits_soft_cap", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "0.0", + "annotation": "float" + }, + { + "name": "zero_tensors", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "cu_seqlens_q_padded", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "cu_seqlens_k_padded", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "s_aux", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "kwargs", + "kind": "VAR_KEYWORD", + "default": null, + "annotation": null + } + ], + "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]" + } + }, + { + "name": "flash_attn_with_kvcache", + "qualname": "flash_attn_with_kvcache", + "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha", + "kind": "function", + "doc": "FlashAttention against a KV cache, built on the CK split-KV kernel.\n\n``q`` is ``(batch, seqlen_q, nheads, headdim)``. For a contiguous cache,\n``k_cache``/``v_cache`` are ``(batch, seqlen_cache, nheads_k, headdim)`` and\n``cache_seqlens`` (int or ``(batch,)``) is the number of valid cache entries\n*before* appending. If ``k``/``v`` (the new tokens) are given, they are\nwritten into the cache in place. The query attends to the valid prefix\n(``cache_seqlens`` + ``seqlen_q``) with bottom-right causal alignment.\n\nNotes / current limitations:\n- Rotary embedding is not applied here; pass ``q``/``k`` already rotated\n (``rotary_cos``/``rotary_sin`` are rejected).\n- Paged caches are addressed via ``block_table`` (forwarded to the split-KV\n varlen kernel); in-cache append for paged layouts must be done by the\n caller (pass ``k=v=None``).", + "signature": { + "parameters": [ + { + "name": "q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "k_cache", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "v_cache", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "v", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "rotary_cos", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "rotary_sin", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "cache_seqlens", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": null + }, + { + "name": "block_table", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "softmax_scale", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "float | None" + }, + { + "name": "causal", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "True", + "annotation": "bool" + }, + { + "name": "window_size", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "(-1, -1, 0)", + "annotation": "Tuple[int, ...]" + }, + { + "name": "s_aux", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "return_lse", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "kwargs", + "kind": "VAR_KEYWORD", + "default": null, + "annotation": null + } + ], + "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]" + } + } + ], + "layers": [] +} diff --git a/build/torch211-rocm72-x86_64-linux/__init__.py b/build/torch211-rocm72-x86_64-linux/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..73acb58c1142a1ab2a6ed9d6f34bc6a53ab5e9b5 --- /dev/null +++ b/build/torch211-rocm72-x86_64-linux/__init__.py @@ -0,0 +1,27 @@ +"""AITER Composable-Kernel FlashAttention kernels for AMD ROCm (compiled HIP). + +Compiled (Composable Kernel / HIP) counterpart to the Triton-based +``aiter-flash-attn`` package, repackaged from the CK FMHA forward path of the +`ROCm/aiter `_ project as a self-contained +Hugging Face Hub kernel. Exposes the FlashAttention forward entry points with +learnable attention sink support (e.g. gpt-oss). +""" + +from .mha import ( + flash_attn_func, + flash_attn_varlen_func, + flash_attn_with_kvcache, +) + + +__kernel_metadata__ = { + "license": "mit", +} + + +__all__ = [ + "__kernel_metadata__", + "flash_attn_func", + "flash_attn_varlen_func", + "flash_attn_with_kvcache", +] diff --git a/build/torch211-rocm72-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so b/build/torch211-rocm72-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so new file mode 100644 index 0000000000000000000000000000000000000000..54937bd77f1e79a0fafa7a1694d1b3b07e165267 --- /dev/null +++ b/build/torch211-rocm72-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c2f41c2913536638920d0d10d5ab4b4337639df8803362367d4edde7020b610 +size 192185696 diff --git a/build/torch211-rocm72-x86_64-linux/_ops.py b/build/torch211-rocm72-x86_64-linux/_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..2b4dd56c220b61fd2c16e0c1a02ee91a491cef0c --- /dev/null +++ b/build/torch211-rocm72-x86_64-linux/_ops.py @@ -0,0 +1,9 @@ +import torch +from . import _aiter_flash_attn_ck_rocm_fe2bb66 +ops = torch.ops._aiter_flash_attn_ck_rocm_fe2bb66 + +def add_op_namespace_prefix(op_name: str): + """ + Prefix op by namespace. + """ + return f"_aiter_flash_attn_ck_rocm_fe2bb66::{op_name}" diff --git a/build/torch211-rocm72-x86_64-linux/metadata.json b/build/torch211-rocm72-x86_64-linux/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..36ab1270c4ce854bebd2f527c6daab14a07c0cb6 --- /dev/null +++ b/build/torch211-rocm72-x86_64-linux/metadata.json @@ -0,0 +1,37 @@ +{ + "name": "aiter-flash-attn-ck", + "id": "_aiter_flash_attn_ck_rocm_fe2bb66", + "version": 1, + "kernels-minver": "0.14.0", + "license": "MIT", + "python-depends": [], + "kernel-depends": [], + "backend": { + "type": "rocm", + "archs": [ + "gfx942", + "gfx950" + ] + }, + "digest": { + "algorithm": "sha256", + "files": { + "__init__.py": "7wj1GTm2bqTLLCyT4xLuAk/+JWYrhINjY/ODMNWhgZ0=", + "_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so": "PC9BwpE1NmOJINDRDVq0tDN2Od+IAzYjZ9Tt3nAgthA=", + "_ops.py": "4WkdYDWWJp60zep+oW/5ym9vIeVMNfrgRCkP3o/w7yU=", + "mha.py": "zCAjY4O+DiCzmEL+W9sqXxZaqjsj4RJbWFcyPpUZejk=", + "symbols.json": "rhnF4H8Jsb6I+YPxIVFMgnLcTQJODEXbZ7ge77NZXAU=" + } + }, + "provenance": { + "kernel-builder": { + "version": "0.18.0-dev0", + "commit": "d054906a13710b811b96697b4635f03020842733", + "dirty": false + }, + "kernel": { + "commit": "fe2bb66fee6c2a77eccf08a0508dc57428d488ed", + "dirty": false + } + } +} \ No newline at end of file diff --git a/build/torch211-rocm72-x86_64-linux/metadata.json.sigstore b/build/torch211-rocm72-x86_64-linux/metadata.json.sigstore new file mode 100644 index 0000000000000000000000000000000000000000..278b5c5efafb50535c26a4f18f466df5dafbf8ee --- /dev/null +++ b/build/torch211-rocm72-x86_64-linux/metadata.json.sigstore @@ -0,0 +1 @@ +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json", "verificationMaterial":{"certificate":{"rawBytes":"MIIHKzCCBrGgAwIBAgIUQTtf3Iyvqs7McLpN4yWfDq9ATiUwCgYIKoZIzj0EAwMwNzEVMBMGA1UEChMMc2lnc3RvcmUuZGV2MR4wHAYDVQQDExVzaWdzdG9yZS1pbnRlcm1lZGlhdGUwHhcNMjYxMDE5MjIzMTE0WhcNMjYxMDE5MjI0MTE0WjAAMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE3t4nw4ZHrBQ3CbTfkKAwLntof/04udWnGPHoBy22SWv0MFnWlFpH3ZxJg5xRTPsjhFBq+fbV9SsvCvDT14AWx6OCBdAwggXMMA4GA1UdDwEB/wQEAwIHgDATBgNVHSUEDDAKBggrBgEFBQcDAzAfBgNVHSMEGDAWgBTf0+nPViQRlvmo2OkoVaLGLhhkPzBrBgNVHREBAf8EYTBfhl1odHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkvLmdpdGh1Yi93b3JrZmxvd3MvYnVpbGQueWFtbEByZWZzL2hlYWRzL21haW4wOQYKKwYBBAGDvzABAQQraHR0cHM6Ly90b2tlbi5hY3Rpb25zLmdpdGh1YnVzZXJjb250ZW50LmNvbTAfBgorBgEEAYO/MAECBBF3b3JrZmxvd19kaXNwYXRjaDA2BgorBgEEAYO/MAEDBChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMBMGCisGAQQBg78wAQQEBUJ1aWxkMCsGCisGAQQBg78wAQUEHWh1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5MB0GCisGAQQBg78wAQYED3JlZnMvaGVhZHMvbWFpbjA7BgorBgEEAYO/MAEIBC0MK2h0dHBzOi8vdG9rZW4uYWN0aW9ucy5naXRodWJ1c2VyY29udGVudC5jb20wbQYKKwYBBAGDvzABCQRfDF1odHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkvLmdpdGh1Yi93b3JrZmxvd3MvYnVpbGQueWFtbEByZWZzL2hlYWRzL21haW4wOAYKKwYBBAGDvzABCgQqDChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMBsGCisGAQQBg78wAQsEDQwLc2VsZi1ob3N0ZWQwQAYKKwYBBAGDvzABDAQyDDBodHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkwOAYKKwYBBAGDvzABDQQqDChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMB8GCisGAQQBg78wAQ4EEQwPcmVmcy9oZWFkcy9tYWluMBoGCisGAQQBg78wAQ8EDAwKMTA3MTQ3NTUyOTAuBgorBgEEAYO/MAEQBCAMHmh0dHBzOi8vZ2l0aHViLmNvbS9odWdnaW5nZmFjZTAYBgorBgEEAYO/MAERBAoMCDI1NzIwNzQzMG0GCisGAQQBg78wARIEXwxdaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5Ly5naXRodWIvd29ya2Zsb3dzL2J1aWxkLnlhbWxAcmVmcy9oZWFkcy9tYWluMDgGCisGAQQBg78wARMEKgwoZmUyYmI2NmZlZTZjMmE3N2VjY2YwOGEwNTA4ZGM1NzQyOGQ0ODhlZDAhBgorBgEEAYO/MAEUBBMMEXdvcmtmbG93X2Rpc3BhdGNoMGQGCisGAQQBg78wARUEVgxUaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5L2FjdGlvbnMvcnVucy8zOTIzOTUxODEwNC9hdHRlbXB0cy8xMBYGCisGAQQBg78wARYECAwGcHVibGljMEYGCisGAQQBg78wARgEOAw2cmVwbzpodWdnaW5nZmFjZS9rZXJuZWxzLWNvbW11bml0eTpyZWY6cmVmcy9oZWFkcy9tYWluMIGJBgorBgEEAdZ5AgQCBHsEeQB3AHUA3T0wasbHETJjGR4cmWc3AqJKXrjePK3/h4pygC8p7o4AAAGhVkpUpwAABAMARjBEAiA1C1YmkJfAITRpzy5HtV9VCMYhGsoyd5hZgpWPpvfVKQIgOVIL0hfBTYj6gt4xl1xqgUEMiH5fkUfg3ZqFxFEBSRAwCgYIKoZIzj0EAwMDaAAwZQIxAJa4QXsPL81L+vXhGGcrZ1zELYnOUe8fmWuz/GLBPTq/v9rdnXKIzxp1A6bIWU8zowIwHGaQvnlFluvNCg6iFIDlDH///B5I/d+ZJwxA3O9O0q7tZggFRCSR0TX4b4K0pfeQ"}, "tlogEntries":[{"logIndex":"3327211583", "logId":{"keyId":"wNI9atQGlz+VWfO6LRygH4QUfY/8W4RFwiT5i5WRgB0="}, "kindVersion":{"kind":"hashedrekord", "version":"0.0.1"}, "integratedTime":"1792449074", "inclusionPromise":{"signedEntryTimestamp":"MEUCIAr7jCNtRJxMlzr1cB+bYl7IKogmxEEEVe2YNbLgJHA8AiEAiDYLQfiwGECJu77/mDsLBPN+ENXHui3Cb6O8P9ck/H0="}, "inclusionProof":{"logIndex":"3205307321", "rootHash":"+fepLxpPOkncVXhGVS4DEMtC08T30oRyfG2VFcMxDGk=", "treeSize":"3205307355", "hashes":["c7VQLPQB2GvvWkc+o6TggbQrbw9Fpr+eCKJewVrBtS8=", "N+iK1dAfNviVQ3nNHztMlyqFa74NPSlTSceuPLmAmTw=", "2wpyyp1ow0otcAGP3cCZwWLmF9FNsuEklXqugfMzXy8=", "JCSa700+gdDu+84caCYaG3zOO7mrMW3EPEh9Uwh91gU=", "mmy8PbTWegE54CQglLSVqGow9D3xL3rqWVCm4og3U2E=", "ZFFNXt17Kn8H1nfr6Yedo+Gde+FkiTYwyKxIfTBJEhs=", "kb9vIWaHg98rThNcZN0YfYNvf4CBr8OBzgh2Qa/FH2A=", "ykqNcyTAH4N/u/MuimGrj8+/P23oqcyJgXQ8Hd4uapU=", "AJlvaM7WQKZg0FAjSdh7RzJXyGKaOW5DVz46FbaUPZE=", "t4Yy4kaoiJ3+VosqXw5Ogmg+A7p/E6y8+JhqWbalMbI=", "5egUlZ4Pi0Ix/qLyCDXocsWywJCHqrde/S06vOZvlOM=", "QsXufUpoKtSjKAB19K1G00umQpWvPHlBNuRGFiiVjQQ=", "QIMkwQbwAewhLJd//5FFwI3SnyNQBGjCgJrelfG8vhc=", "Et17CACUv9QN64irUJuBjsrgRGkGnnJ9xacRV/aBdwQ=", "lqgEwkxhDMs/gdIAE4lmjfMwg7nDmv7lda6O0LGliCw=", "HovTPBWzoErvUg1u5sIi3x1ZonZ7yJekVsBrmpomCIA=", "G+nk7mgE5h6dUmv2UY7NDJrY3WvrZY31raaO1AETt+o=", "qlTWNDzERnTBQVcHHXXVPjiUP0SsikGxfcxVlgqVFDo=", "gg9GNApNke3+zNhl4tsWtmIycCFrOMCB3lys6XuSQNQ=", "NlOclldmKPA1tMKIskqHAwuQDYHp31YwpK+OOMrMccc=", "qxzHanAzz57SDdmJe0B7bJK72NTIBbwEMGBBKvDOROw=", "xH/DCseLHr9eKoYT8qsORZK7zVdEGYWHuVtsVrD95wY="], "checkpoint":{"envelope":"rekor.sigstore.dev - 1193050959916656506\n3205307355\n+fepLxpPOkncVXhGVS4DEMtC08T30oRyfG2VFcMxDGk=\n\n— rekor.sigstore.dev wNI9ajBEAiBdd0/8rQRLHE4cgX8ENMZ0/RCQPKa4qsZYGkjcthEV3AIgf8uLH7bF4jGMwAoMMQPLM1CGWjMYNWqJw3068ntJIiY=\n"}}, "canonicalizedBody":"eyJhcGlWZXJzaW9uIjoiMC4wLjEiLCJraW5kIjoiaGFzaGVkcmVrb3JkIiwic3BlYyI6eyJkYXRhIjp7Imhhc2giOnsiYWxnb3JpdGhtIjoic2hhMjU2IiwidmFsdWUiOiJjOGJkZjdhN2I0MWRlNTdiZDMyY2Q4NzdhNWRmODNkOTJkNTk2Y2MwMWYzYTM2ZTE1NTEyMjIxZjA4NWU2NGYxIn19LCJzaWduYXR1cmUiOnsiY29udGVudCI6Ik1FVUNJUUN6cEsrZGZSeXdLV3RGVVZHVDB5RXFkcDBOdnNBRXU2UEtLWWVNbm0wOXFRSWdJVmNVcUVnUmVJcDdtd0NMOEFsTkFocThZL3hvL012c09mV2RiQ3Z1QVlZPSIsInB1YmxpY0tleSI6eyJjb250ZW50IjoiTFMwdExTMUNSVWRKVGlCRFJWSlVTVVpKUTBGVVJTMHRMUzB0Q2sxSlNVaExla05EUW5KSFowRjNTVUpCWjBsVlVWUjBaak5KZVhaeGN6ZE5ZMHh3VGpSNVYyWkVjVGxCVkdsVmQwTm5XVWxMYjFwSmVtb3dSVUYzVFhjS1RucEZWazFDVFVkQk1WVkZRMmhOVFdNeWJHNWpNMUoyWTIxVmRWcEhWakpOVWpSM1NFRlpSRlpSVVVSRmVGWjZZVmRrZW1SSE9YbGFVekZ3WW01U2JBcGpiVEZzV2tkc2FHUkhWWGRJYUdOT1RXcFplRTFFUlRWTmFrbDZUVlJGTUZkb1kwNU5hbGw0VFVSRk5VMXFTVEJOVkVVd1YycEJRVTFHYTNkRmQxbElDa3R2V2tsNmFqQkRRVkZaU1V0dldrbDZhakJFUVZGalJGRm5RVVV6ZERSdWR6UmFTSEpDVVRORFlsUm1hMHRCZDB4dWRHOW1MekEwZFdSWGJrZFFTRzhLUW5reU1sTlhkakJOUm01WGJFWndTRE5hZUVwbk5YaFNWRkJ6YW1oR1FuRXJabUpXT1ZOemRrTjJSRlF4TkVGWGVEWlBRMEprUVhkbloxaE5UVUUwUndwQk1WVmtSSGRGUWk5M1VVVkJkMGxJWjBSQlZFSm5UbFpJVTFWRlJFUkJTMEpuWjNKQ1owVkdRbEZqUkVGNlFXWkNaMDVXU0ZOTlJVZEVRVmRuUWxSbUNqQXJibEJXYVZGU2JIWnRiekpQYTI5V1lVeEhUR2hvYTFCNlFuSkNaMDVXU0ZKRlFrRm1PRVZaVkVKbWFHd3hiMlJJVW5kamVtOTJUREprY0dSSGFERUtXV2sxYW1JeU1IWmhTRlp1V2pKc2RWb3lXbWhaTWxWMllUSldlV0p0Vm5OamVURnFZakl4ZEdSWE5YQmtTR3QyVEcxa2NHUkhhREZaYVRrellqTktjZ3BhYlhoMlpETk5kbGx1Vm5CaVIxRjFaVmRHZEdKRlFubGFWMXA2VERKb2JGbFhVbnBNTWpGb1lWYzBkMDlSV1V0TGQxbENRa0ZIUkhaNlFVSkJVVkZ5Q21GSVVqQmpTRTAyVEhrNU1HSXlkR3hpYVRWb1dUTlNjR0l5TlhwTWJXUndaRWRvTVZsdVZucGFXRXBxWWpJMU1GcFhOVEJNYlU1MllsUkJaa0puYjNJS1FtZEZSVUZaVHk5TlFVVkRRa0pHTTJJelNuSmFiWGgyWkRFNWEyRllUbmRaV0ZKcVlVUkJNa0puYjNKQ1owVkZRVmxQTDAxQlJVUkNRMmh0V2xSS2FRcFphbGt5V20xV2JFNXRUWGxaVkdNeldsZE9hbHBxUVRSWlZFRXhUVVJvYTFsNlZUTk9SRWswV2tSUk5FOUhWbXROUWsxSFEybHpSMEZSVVVKbk56aDNDa0ZSVVVWQ1ZVb3hZVmQ0YTAxRGMwZERhWE5IUVZGUlFtYzNPSGRCVVZWRlNGZG9NVm95WkhCaWJXUnRXVmRPYkV3eWRHeGpiVFZzWWtoTmRGa3lPWFFLWWxoV2RXRllValZOUWpCSFEybHpSMEZSVVVKbk56aDNRVkZaUlVRelNteGFiazEyWVVkV2FGcElUWFppVjBad1ltcEJOMEpuYjNKQ1owVkZRVmxQTHdwTlFVVkpRa013VFVzeWFEQmtTRUo2VDJrNGRtUkhPWEphVnpSMVdWZE9NR0ZYT1hWamVUVnVZVmhTYjJSWFNqRmpNbFo1V1RJNWRXUkhWblZrUXpWcUNtSXlNSGRpVVZsTFMzZFpRa0pCUjBSMmVrRkNRMUZTWmtSR01XOWtTRkozWTNwdmRrd3laSEJrUjJneFdXazFhbUl5TUhaaFNGWnVXakpzZFZveVdtZ0tXVEpWZG1FeVZubGliVlp6WTNreGFtSXlNWFJrVnpWd1pFaHJka3h0WkhCa1IyZ3hXV2s1TTJJelNuSmFiWGgyWkROTmRsbHVWbkJpUjFGMVpWZEdkQXBpUlVKNVdsZGFla3d5YUd4WlYxSjZUREl4YUdGWE5IZFBRVmxMUzNkWlFrSkJSMFIyZWtGQ1EyZFJjVVJEYUcxYVZFcHBXV3BaTWxwdFZteE9iVTE1Q2xsVVl6TmFWMDVxV21wQk5GbFVRVEZOUkdocldYcFZNMDVFU1RSYVJGRTBUMGRXYTAxQ2MwZERhWE5IUVZGUlFtYzNPSGRCVVhORlJGRjNUR015Vm5NS1dta3hiMkl6VGpCYVYxRjNVVUZaUzB0M1dVSkNRVWRFZG5wQlFrUkJVWGxFUkVKdlpFaFNkMk42YjNaTU1tUndaRWRvTVZscE5XcGlNakIyWVVoV2JncGFNbXgxV2pKYWFGa3lWWFpoTWxaNVltMVdjMk41TVdwaU1qRjBaRmMxY0dSSWEzZFBRVmxMUzNkWlFrSkJSMFIyZWtGQ1JGRlJjVVJEYUcxYVZFcHBDbGxxV1RKYWJWWnNUbTFOZVZsVVl6TmFWMDVxV21wQk5GbFVRVEZOUkdocldYcFZNMDVFU1RSYVJGRTBUMGRXYTAxQ09FZERhWE5IUVZGUlFtYzNPSGNLUVZFMFJVVlJkMUJqYlZadFkzazViMXBYUm10amVUbDBXVmRzZFUxQ2IwZERhWE5IUVZGUlFtYzNPSGRCVVRoRlJFRjNTMDFVUVROTlZGRXpUbFJWZVFwUFZFRjFRbWR2Y2tKblJVVkJXVTh2VFVGRlVVSkRRVTFJYldnd1pFaENlazlwT0haYU1td3dZVWhXYVV4dFRuWmlVemx2WkZka2JtRlhOVzVhYlVacUNscFVRVmxDWjI5eVFtZEZSVUZaVHk5TlFVVlNRa0Z2VFVORVNURk9la2wzVG5wUmVrMUhNRWREYVhOSFFWRlJRbWMzT0hkQlVrbEZXSGQ0WkdGSVVqQUtZMGhOTmt4NU9XNWhXRkp2WkZkSmRWa3lPWFJNTW1neFdqSmtjR0p0WkcxWlYwNXNUREowYkdOdE5XeGlTRTEwV1RJNWRHSllWblZoV0ZJMVRIazFiZ3BoV0ZKdlpGZEpkbVF5T1hsaE1scHpZak5rZWt3eVNqRmhWM2hyVEc1c2FHSlhlRUZqYlZadFkzazViMXBYUm10amVUbDBXVmRzZFUxRVowZERhWE5IQ2tGUlVVSm5OemgzUVZKTlJVdG5kMjlhYlZWNVdXMUpNazV0V214YVZGcHFUVzFGTTA0eVZtcFpNbGwzVDBkRmQwNVVRVFJhUjAweFRucFJlVTlIVVRBS1QwUm9iRnBFUVdoQ1oyOXlRbWRGUlVGWlR5OU5RVVZWUWtKTlRVVllaSFpqYlhSdFlrYzVNMWd5VW5Cak0wSm9aRWRPYjAxSFVVZERhWE5IUVZGUlFncG5OemgzUVZKVlJWWm5lRlZoU0ZJd1kwaE5Oa3g1T1c1aFdGSnZaRmRKZFZreU9YUk1NbWd4V2pKa2NHSnRaRzFaVjA1c1RESjBiR050Tld4aVNFMTBDbGt5T1hSaVdGWjFZVmhTTlV3eVJtcGtSMngyWW01TmRtTnVWblZqZVRoNlQxUkplazlVVlhoUFJFVjNUa001YUdSSVVteGlXRUl3WTNrNGVFMUNXVWNLUTJselIwRlJVVUpuTnpoM1FWSlpSVU5CZDBkalNGWnBZa2RzYWsxRldVZERhWE5IUVZGUlFtYzNPSGRCVW1kRlQwRjNNbU50Vm5kaWVuQnZaRmRrYmdwaFZ6VnVXbTFHYWxwVE9YSmFXRXAxV2xkNGVreFhUblppVnpFeFltMXNNR1ZVY0hsYVYxazJZMjFXYldONU9XOWFWMFpyWTNrNWRGbFhiSFZOU1VkS0NrSm5iM0pDWjBWRlFXUmFOVUZuVVVOQ1NITkZaVkZDTTBGSVZVRXpWREIzWVhOaVNFVlVTbXBIVWpSamJWZGpNMEZ4U2t0WWNtcGxVRXN6TDJnMGNIa0taME00Y0Rkdk5FRkJRVWRvVm10d1ZYQjNRVUZDUVUxQlVtcENSVUZwUVRGRE1WbHRhMHBtUVVsVVVuQjZlVFZJZEZZNVZrTk5XV2hIYzI5NVpEVm9XZ3BuY0ZkUWNIWm1Wa3RSU1dkUFZrbE1NR2htUWxSWmFqWm5kRFI0YkRGNGNXZFZSVTFwU0RWbWExVm1aek5hY1VaNFJrVkNVMUpCZDBObldVbExiMXBKQ25wcU1FVkJkMDFFWVVGQmQxcFJTWGhCU21FMFVWaHpVRXc0TVV3cmRsaG9SMGRqY2xveGVrVk1XVzVQVldVNFptMVhkWG92UjB4Q1VGUnhMM1k1Y21RS2JsaExTWHA0Y0RGQk5tSkpWMVU0ZW05M1NYZElSMkZSZG01c1JteDFkazVEWnpacFJrbEViRVJJTHk4dlFqVkpMMlFyV2twM2VFRXpUemxQTUhFM2RBcGFaMmRHVWtOVFVqQlVXRFJpTkVzd2NHWmxVUW90TFMwdExVVk9SQ0JEUlZKVVNVWkpRMEZVUlMwdExTMHRDZz09In19fX0="}], "timestampVerificationData":{"rfc3161Timestamps":[{"signedTimestamp":"MIICyzADAgEAMIICwgYJKoZIhvcNAQcCoIICszCCAq8CAQMxDTALBglghkgBZQMEAgEwgbgGCyqGSIb3DQEJEAEEoIGoBIGlMIGiAgEBBgkrBgEEAYO/MAIwMTANBglghkgBZQMEAgEFAAQgnxpe49Bca8BoOZiHasRIYzPfojWVj2O6LRUdho/SpeQCFQCVCrq01TcmtZOV7ywJOzs2MCHCpBgPMjAyNjEwMTkyMjMxMTRaMAMCAQGgMqQwMC4xFTATBgNVBAoTDHNpZ3N0b3JlLmRldjEVMBMGA1UEAxMMc2lnc3RvcmUtdHNhoAAxggHcMIIB2AIBATBRMDkxFTATBgNVBAoTDHNpZ3N0b3JlLmRldjEgMB4GA1UEAxMXc2lnc3RvcmUtdHNhLXNlbGZzaWduZWQCFDoTVC8MkGHuvMFDL8uKjosqI4sMMAsGCWCGSAFlAwQCAaCB/DAaBgkqhkiG9w0BCQMxDQYLKoZIhvcNAQkQAQQwHAYJKoZIhvcNAQkFMQ8XDTI2MTAxOTIyMzExNFowLwYJKoZIhvcNAQkEMSIEILuVLXt8AmHMN6KRJ5J/f2uWMscqD7w9RGP4/WIFhtaqMIGOBgsqhkiG9w0BCRACLzF/MH0wezB5BCCF+Se8B6tiysO0Q1bBDvyBssaIP9p6uebYcNnROs0FtzBVMD2kOzA5MRUwEwYDVQQKEwxzaWdzdG9yZS5kZXYxIDAeBgNVBAMTF3NpZ3N0b3JlLXRzYS1zZWxmc2lnbmVkAhQ6E1QvDJBh7rzBQy/Lio6LKiOLDDAKBggqhkjOPQQDAgRoMGYCMQDBQ5gG1/t9qM5NWnE0tWE0k7iAjLVh1LkWH5yuTH5gMTyC1peLgRopwwY6jQrkyr4CMQCEct/P2loSuprwJPAduGECrNug2LYMwl+jfNReEoXLVUVcTHdo4UN53D3t/T0WJfw="}]}}, "messageSignature":{"messageDigest":{"algorithm":"SHA2_256", "digest":"yL33p7Qd5XvTLNh3pd+D2S1ZbMAfOjbhVRIiHwheZPE="}, "signature":"MEUCIQCzpK+dfRywKWtFUVGT0yEqdp0NvsAEu6PKKYeMnm09qQIgIVcUqEgReIp7mwCL8AlNAhq8Y/xo/MvsOfWdbCvuAYY="}} \ No newline at end of file diff --git a/build/torch211-rocm72-x86_64-linux/mha.py b/build/torch211-rocm72-x86_64-linux/mha.py new file mode 100644 index 0000000000000000000000000000000000000000..01e2d6a0fbee98d27129f2eb695ac38df73e6487 --- /dev/null +++ b/build/torch211-rocm72-x86_64-linux/mha.py @@ -0,0 +1,332 @@ +"""FlashAttention forward entry points backed by aiter's Composable-Kernel FMHA. + +This is the compiled (HIP / Composable Kernel) counterpart to the Triton +``aiter-flash-attn`` package. Only the *forward* pass is provided (dense and +variable-length), including support for learnable attention sinks via +``window_size[2]`` / ``s_aux`` (e.g. gpt-oss). The op set ships exclusively +the CK ``mha_fwd`` / ``mha_varlen_fwd`` kernels, so the sink path is always +taken (no fall-through to a non-sink ASM kernel). +""" + +from typing import Optional, Tuple, Union + +import torch +from torch import Tensor + +from ._ops import ops + + +def _round_multiple(x: int, m: int) -> int: + return (x + m - 1) // m * m + + +def _maybe_pad_headdim(t: Tensor, target: int) -> Tensor: + og = t.size(-1) + if og == target: + return t + return torch.nn.functional.pad(t, [0, target - og]) + + +def _unpack_window(window_size) -> Tuple[int, int, int]: + left = int(window_size[0]) + right = int(window_size[1]) + sink = int(window_size[2]) if len(window_size) == 3 else 0 + return left, right, sink + + +def _prep_sink(s_aux: Optional[Tensor], nheads: int, device) -> Optional[Tensor]: + if s_aux is None: + return None + assert s_aux.device == device, "s_aux must be on the same device as q" + assert s_aux.shape[0] == nheads, "s_aux must have shape [nheads_q]" + if s_aux.dtype != torch.float32: + s_aux = s_aux.to(torch.float32) + return s_aux + + +def _effective_sink_size(sink_size: int, s_aux: Optional[Tensor]) -> int: + # The CK kernel only takes the sink path when sink_size > 0. Callers using + # the transformers attention interface pass the per-head sink logits as + # ``s_aux`` but leave ``window_size`` a 2-tuple (sink_size == 0), so enable + # a single learnable sink automatically when s_aux is given. + if s_aux is not None and sink_size == 0: + return 1 + return sink_size + + +def flash_attn_func( + q: Tensor, + k: Tensor, + v: Tensor, + dropout_p: float = 0.0, + softmax_scale: Optional[float] = None, + causal: bool = False, + window_size: Tuple[int, ...] = (-1, -1, 0), + bias: Optional[Tensor] = None, + alibi_slopes: Optional[Tensor] = None, + return_lse: bool = False, + return_attn_probs: bool = False, + s_aux: Optional[Tensor] = None, + **kwargs, +) -> Union[Tensor, Tuple[Tensor, ...]]: + """Dense FlashAttention forward. + + ``q, k, v`` have shape ``(batch, seqlen, nheads, headdim)``. ``window_size`` + is ``(left, right, sink_size)``; ``s_aux`` is an optional ``(nheads_q,)`` + fp32 tensor of learnable sink logits. Extra keyword arguments (e.g. those + passed by the transformers flash-attention interface) are ignored. + """ + if softmax_scale is None: + softmax_scale = q.shape[-1] ** (-0.5) + window_size_left, window_size_right, sink_size = _unpack_window(window_size) + + nheads_q = q.size(2) + s_aux = _prep_sink(s_aux, nheads_q, q.device) + sink_size = _effective_sink_size(sink_size, s_aux) + + head_q_og = q.size(-1) + head_v_og = v.size(-1) + head_q = _round_multiple(head_q_og, 8) + head_v = _round_multiple(head_v_og, 8) + q = _maybe_pad_headdim(q, head_q) + k = _maybe_pad_headdim(k, head_q) + v = _maybe_pad_headdim(v, head_v) + + return_softmax = return_attn_probs and dropout_p > 0 + out, softmax_lse, S_dmask, _ = ops.mha_fwd( + q, + k, + v, + dropout_p, + softmax_scale, + causal, + window_size_left, + window_size_right, + sink_size, + return_lse or return_attn_probs, + return_softmax, + None, # cu_seqlens_q + None, # cu_seqlens_kv + None, # out + bias, + alibi_slopes, + None, # q_descale + None, # k_descale + None, # v_descale + s_aux, + None, # gen + ) + + out = out[..., :head_v_og] + if not (return_lse or return_attn_probs): + return out + result = (out,) + if return_lse: + result = result + (softmax_lse,) + if return_attn_probs: + result = result + (S_dmask,) + return result + + +def flash_attn_varlen_func( + q: Tensor, + k: Tensor, + v: Tensor, + cu_seqlens_q: Tensor, + cu_seqlens_k: Tensor, + max_seqlen_q: int, + max_seqlen_k: int, + dropout_p: float = 0.0, + softmax_scale: Optional[float] = None, + causal: bool = False, + window_size: Tuple[int, ...] = (-1, -1, 0), + bias: Optional[Tensor] = None, + alibi_slopes: Optional[Tensor] = None, + return_lse: bool = False, + return_attn_probs: bool = False, + block_table: Optional[Tensor] = None, + min_seqlen_q: int = 0, + logits_soft_cap: float = 0.0, + zero_tensors: bool = False, + cu_seqlens_q_padded: Optional[Tensor] = None, + cu_seqlens_k_padded: Optional[Tensor] = None, + s_aux: Optional[Tensor] = None, + **kwargs, +) -> Union[Tensor, Tuple[Tensor, ...]]: + """Variable-length / packed FlashAttention forward. + + ``q`` has shape ``(total_q, nheads, headdim)``; ``cu_seqlens_*`` are int32 + cumulative offsets of shape ``(batch + 1,)``. Sinks are supported through + ``window_size[2]`` / ``s_aux`` as in :func:`flash_attn_func`. Extra keyword + arguments (e.g. from the transformers flash-attention interface) are ignored. + """ + if softmax_scale is None: + softmax_scale = q.shape[-1] ** (-0.5) + window_size_left, window_size_right, sink_size = _unpack_window(window_size) + + nheads_q = q.size(1) + s_aux = _prep_sink(s_aux, nheads_q, q.device) + sink_size = _effective_sink_size(sink_size, s_aux) + + head_q_og = q.size(-1) + head_v_og = v.size(-1) + head_q = _round_multiple(head_q_og, 8) + head_v = _round_multiple(head_v_og, 8) + q = _maybe_pad_headdim(q, head_q) + k = _maybe_pad_headdim(k, head_q) + v = _maybe_pad_headdim(v, head_v) + + return_softmax = return_attn_probs and dropout_p > 0 + out, softmax_lse, S_dmask, _ = ops.mha_varlen_fwd( + q, + k, + v, + cu_seqlens_q, + cu_seqlens_k, + max_seqlen_q, + max_seqlen_k, + min_seqlen_q, + dropout_p, + softmax_scale, + logits_soft_cap, + zero_tensors, + causal, + window_size_left, + window_size_right, + sink_size, + return_lse or return_attn_probs, + return_softmax, + None, # out + block_table, + bias, + alibi_slopes, + None, # q_descale + None, # k_descale + None, # v_descale + None, # gen + cu_seqlens_q_padded, + cu_seqlens_k_padded, + s_aux, + ) + + out = out[..., :head_v_og] + if not (return_lse or return_attn_probs): + return out + result = (out,) + if return_lse: + result = result + (softmax_lse,) + if return_attn_probs: + result = result + (S_dmask,) + return result + + +def _append_to_contiguous_cache(k_cache, v_cache, k, v, cache_seqlens, Sq): + """In-place append of new k/v into a contiguous cache at per-batch offsets.""" + B = k_cache.shape[0] + if isinstance(cache_seqlens, int): + cache_seqlens = torch.full((B,), cache_seqlens, device=k_cache.device, dtype=torch.long) + pos = cache_seqlens.view(B, 1).to(torch.long) + torch.arange(Sq, device=k_cache.device).view(1, Sq) + bidx = torch.arange(B, device=k_cache.device).view(B, 1).expand(B, Sq) + k_cache[bidx, pos] = k.to(k_cache.dtype) + v_cache[bidx, pos] = v.to(v_cache.dtype) + + +def flash_attn_with_kvcache( + q: Tensor, + k_cache: Tensor, + v_cache: Tensor, + k: Optional[Tensor] = None, + v: Optional[Tensor] = None, + rotary_cos: Optional[Tensor] = None, + rotary_sin: Optional[Tensor] = None, + cache_seqlens=None, + block_table: Optional[Tensor] = None, + softmax_scale: Optional[float] = None, + causal: bool = True, + window_size: Tuple[int, ...] = (-1, -1, 0), + s_aux: Optional[Tensor] = None, + return_lse: bool = False, + **kwargs, +) -> Union[Tensor, Tuple[Tensor, ...]]: + """FlashAttention against a KV cache, built on the CK split-KV kernel. + + ``q`` is ``(batch, seqlen_q, nheads, headdim)``. For a contiguous cache, + ``k_cache``/``v_cache`` are ``(batch, seqlen_cache, nheads_k, headdim)`` and + ``cache_seqlens`` (int or ``(batch,)``) is the number of valid cache entries + *before* appending. If ``k``/``v`` (the new tokens) are given, they are + written into the cache in place. The query attends to the valid prefix + (``cache_seqlens`` + ``seqlen_q``) with bottom-right causal alignment. + + Notes / current limitations: + - Rotary embedding is not applied here; pass ``q``/``k`` already rotated + (``rotary_cos``/``rotary_sin`` are rejected). + - Paged caches are addressed via ``block_table`` (forwarded to the split-KV + varlen kernel); in-cache append for paged layouts must be done by the + caller (pass ``k=v=None``). + """ + if rotary_cos is not None or rotary_sin is not None: + raise NotImplementedError( + "flash_attn_with_kvcache does not apply rotary; rotate q/k beforehand." + ) + if softmax_scale is None: + softmax_scale = q.shape[-1] ** (-0.5) + window_size_left, window_size_right, sink_size = _unpack_window(window_size) + B, Sq, Hq, D = q.shape + s_aux = _prep_sink(s_aux, Hq, q.device) + sink_size = _effective_sink_size(sink_size, s_aux) + + paged = block_table is not None + + # ---- append new tokens into the cache (contiguous layout only) ---- + if k is not None and not paged: + assert cache_seqlens is not None, "cache_seqlens required to append into the cache" + _append_to_contiguous_cache(k_cache, v_cache, k, v, cache_seqlens, Sq) + elif k is not None and paged: + raise NotImplementedError( + "in-cache append for paged layouts is not supported; append externally " + "and call with k=v=None." + ) + + appended = 0 if k is None else Sq + + # ---- key lengths ---- + if cache_seqlens is None: + klens = torch.full((B,), k_cache.shape[1], device=q.device, dtype=torch.long) + elif isinstance(cache_seqlens, int): + klens = torch.full((B,), cache_seqlens + appended, device=q.device, dtype=torch.long) + else: + klens = cache_seqlens.to(q.device, torch.long) + appended + + uniform = bool((klens == klens[0]).all().item()) + + # ---- fast path: contiguous cache, uniform length -> dense kernel ---- + if not paged and uniform: + L = int(klens[0].item()) + return flash_attn_func( + q, k_cache[:, :L], v_cache[:, :L], + softmax_scale=softmax_scale, causal=causal, + window_size=(window_size_left, window_size_right, sink_size), + s_aux=s_aux, return_lse=return_lse, + ) + + # ---- general path: variable cache lengths -> split-KV varlen kernel ---- + cu_q = torch.arange(0, (B + 1) * Sq, Sq, device=q.device, dtype=torch.int32) + cu_k = torch.zeros(B + 1, device=q.device, dtype=torch.int32) + cu_k[1:] = torch.cumsum(klens, 0).to(torch.int32) + q_packed = q.reshape(B * Sq, Hq, D) + if paged: + k_packed, v_packed = k_cache, v_cache # kernel gathers via block_table + else: + k_packed = torch.cat([k_cache[b, : int(klens[b])] for b in range(B)], dim=0) + v_packed = torch.cat([v_cache[b, : int(klens[b])] for b in range(B)], dim=0) + + out = flash_attn_varlen_func( + q_packed, k_packed, v_packed, cu_q, cu_k, Sq, int(klens.max().item()), + softmax_scale=softmax_scale, causal=causal, + window_size=(window_size_left, window_size_right, sink_size), + block_table=block_table, s_aux=s_aux, return_lse=return_lse, + ) + if return_lse: + out, lse = out + return out.reshape(B, Sq, Hq, D), lse + return out.reshape(B, Sq, Hq, D) diff --git a/build/torch211-rocm72-x86_64-linux/symbols.json b/build/torch211-rocm72-x86_64-linux/symbols.json new file mode 100644 index 0000000000000000000000000000000000000000..e8348db4bd94712352e7fb3e83b1d94f47c2814b --- /dev/null +++ b/build/torch211-rocm72-x86_64-linux/symbols.json @@ -0,0 +1,349 @@ +{ + "schema_version": 1, + "module": "_aiter_flash_attn_ck_rocm_fe2bb66", + "functions": [ + { + "name": "flash_attn_func", + "qualname": "flash_attn_func", + "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha", + "kind": "function", + "doc": "Dense FlashAttention forward.\n\n``q, k, v`` have shape ``(batch, seqlen, nheads, headdim)``. ``window_size``\nis ``(left, right, sink_size)``; ``s_aux`` is an optional ``(nheads_q,)``\nfp32 tensor of learnable sink logits. Extra keyword arguments (e.g. those\npassed by the transformers flash-attention interface) are ignored.", + "signature": { + "parameters": [ + { + "name": "q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "v", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "dropout_p", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "0.0", + "annotation": "float" + }, + { + "name": "softmax_scale", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "float | None" + }, + { + "name": "causal", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "window_size", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "(-1, -1, 0)", + "annotation": "Tuple[int, ...]" + }, + { + "name": "bias", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "alibi_slopes", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "return_lse", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "return_attn_probs", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "s_aux", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "kwargs", + "kind": "VAR_KEYWORD", + "default": null, + "annotation": null + } + ], + "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]" + } + }, + { + "name": "flash_attn_varlen_func", + "qualname": "flash_attn_varlen_func", + "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha", + "kind": "function", + "doc": "Variable-length / packed FlashAttention forward.\n\n``q`` has shape ``(total_q, nheads, headdim)``; ``cu_seqlens_*`` are int32\ncumulative offsets of shape ``(batch + 1,)``. Sinks are supported through\n``window_size[2]`` / ``s_aux`` as in :func:`flash_attn_func`. Extra keyword\narguments (e.g. from the transformers flash-attention interface) are ignored.", + "signature": { + "parameters": [ + { + "name": "q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "v", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "cu_seqlens_q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "cu_seqlens_k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "max_seqlen_q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "int" + }, + { + "name": "max_seqlen_k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "int" + }, + { + "name": "dropout_p", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "0.0", + "annotation": "float" + }, + { + "name": "softmax_scale", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "float | None" + }, + { + "name": "causal", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "window_size", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "(-1, -1, 0)", + "annotation": "Tuple[int, ...]" + }, + { + "name": "bias", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "alibi_slopes", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "return_lse", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "return_attn_probs", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "block_table", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "min_seqlen_q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "0", + "annotation": "int" + }, + { + "name": "logits_soft_cap", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "0.0", + "annotation": "float" + }, + { + "name": "zero_tensors", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "cu_seqlens_q_padded", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "cu_seqlens_k_padded", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "s_aux", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "kwargs", + "kind": "VAR_KEYWORD", + "default": null, + "annotation": null + } + ], + "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]" + } + }, + { + "name": "flash_attn_with_kvcache", + "qualname": "flash_attn_with_kvcache", + "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha", + "kind": "function", + "doc": "FlashAttention against a KV cache, built on the CK split-KV kernel.\n\n``q`` is ``(batch, seqlen_q, nheads, headdim)``. For a contiguous cache,\n``k_cache``/``v_cache`` are ``(batch, seqlen_cache, nheads_k, headdim)`` and\n``cache_seqlens`` (int or ``(batch,)``) is the number of valid cache entries\n*before* appending. If ``k``/``v`` (the new tokens) are given, they are\nwritten into the cache in place. The query attends to the valid prefix\n(``cache_seqlens`` + ``seqlen_q``) with bottom-right causal alignment.\n\nNotes / current limitations:\n- Rotary embedding is not applied here; pass ``q``/``k`` already rotated\n (``rotary_cos``/``rotary_sin`` are rejected).\n- Paged caches are addressed via ``block_table`` (forwarded to the split-KV\n varlen kernel); in-cache append for paged layouts must be done by the\n caller (pass ``k=v=None``).", + "signature": { + "parameters": [ + { + "name": "q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "k_cache", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "v_cache", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "v", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "rotary_cos", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "rotary_sin", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "cache_seqlens", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": null + }, + { + "name": "block_table", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "softmax_scale", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "float | None" + }, + { + "name": "causal", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "True", + "annotation": "bool" + }, + { + "name": "window_size", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "(-1, -1, 0)", + "annotation": "Tuple[int, ...]" + }, + { + "name": "s_aux", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "return_lse", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "kwargs", + "kind": "VAR_KEYWORD", + "default": null, + "annotation": null + } + ], + "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]" + } + } + ], + "layers": [] +} diff --git a/build/torch212-rocm71-x86_64-linux/__init__.py b/build/torch212-rocm71-x86_64-linux/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..73acb58c1142a1ab2a6ed9d6f34bc6a53ab5e9b5 --- /dev/null +++ b/build/torch212-rocm71-x86_64-linux/__init__.py @@ -0,0 +1,27 @@ +"""AITER Composable-Kernel FlashAttention kernels for AMD ROCm (compiled HIP). + +Compiled (Composable Kernel / HIP) counterpart to the Triton-based +``aiter-flash-attn`` package, repackaged from the CK FMHA forward path of the +`ROCm/aiter `_ project as a self-contained +Hugging Face Hub kernel. Exposes the FlashAttention forward entry points with +learnable attention sink support (e.g. gpt-oss). +""" + +from .mha import ( + flash_attn_func, + flash_attn_varlen_func, + flash_attn_with_kvcache, +) + + +__kernel_metadata__ = { + "license": "mit", +} + + +__all__ = [ + "__kernel_metadata__", + "flash_attn_func", + "flash_attn_varlen_func", + "flash_attn_with_kvcache", +] diff --git a/build/torch212-rocm71-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so b/build/torch212-rocm71-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so new file mode 100644 index 0000000000000000000000000000000000000000..037d16362e5c959813f02600f501055cc33d4fa6 --- /dev/null +++ b/build/torch212-rocm71-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78095dfb6de1a2d229d40cdf34e125605bb33d7fa467134d236cb51df66eaf9b +size 201962680 diff --git a/build/torch212-rocm71-x86_64-linux/_ops.py b/build/torch212-rocm71-x86_64-linux/_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..2b4dd56c220b61fd2c16e0c1a02ee91a491cef0c --- /dev/null +++ b/build/torch212-rocm71-x86_64-linux/_ops.py @@ -0,0 +1,9 @@ +import torch +from . import _aiter_flash_attn_ck_rocm_fe2bb66 +ops = torch.ops._aiter_flash_attn_ck_rocm_fe2bb66 + +def add_op_namespace_prefix(op_name: str): + """ + Prefix op by namespace. + """ + return f"_aiter_flash_attn_ck_rocm_fe2bb66::{op_name}" diff --git a/build/torch212-rocm71-x86_64-linux/metadata.json b/build/torch212-rocm71-x86_64-linux/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..7ec030380786a300fd2950a5268ba69efd35b8d8 --- /dev/null +++ b/build/torch212-rocm71-x86_64-linux/metadata.json @@ -0,0 +1,37 @@ +{ + "name": "aiter-flash-attn-ck", + "id": "_aiter_flash_attn_ck_rocm_fe2bb66", + "version": 1, + "kernels-minver": "0.14.0", + "license": "MIT", + "python-depends": [], + "kernel-depends": [], + "backend": { + "type": "rocm", + "archs": [ + "gfx942", + "gfx950" + ] + }, + "digest": { + "algorithm": "sha256", + "files": { + "__init__.py": "7wj1GTm2bqTLLCyT4xLuAk/+JWYrhINjY/ODMNWhgZ0=", + "_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so": "eAld+23hotIp1AzfNOElYFuzPX+kZxNNI2y1HfZur5s=", + "_ops.py": "4WkdYDWWJp60zep+oW/5ym9vIeVMNfrgRCkP3o/w7yU=", + "mha.py": "zCAjY4O+DiCzmEL+W9sqXxZaqjsj4RJbWFcyPpUZejk=", + "symbols.json": "rhnF4H8Jsb6I+YPxIVFMgnLcTQJODEXbZ7ge77NZXAU=" + } + }, + "provenance": { + "kernel-builder": { + "version": "0.18.0-dev0", + "commit": "d054906a13710b811b96697b4635f03020842733", + "dirty": false + }, + "kernel": { + "commit": "fe2bb66fee6c2a77eccf08a0508dc57428d488ed", + "dirty": false + } + } +} \ No newline at end of file diff --git a/build/torch212-rocm71-x86_64-linux/metadata.json.sigstore b/build/torch212-rocm71-x86_64-linux/metadata.json.sigstore new file mode 100644 index 0000000000000000000000000000000000000000..b2b4648e7c49c0a033384f395a9dc7e92265c8db --- /dev/null +++ b/build/torch212-rocm71-x86_64-linux/metadata.json.sigstore @@ -0,0 +1 @@ +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json", "verificationMaterial":{"certificate":{"rawBytes":"MIIHKzCCBrKgAwIBAgIUNCpKwqkj9HF7/3kzrniINKWzzUMwCgYIKoZIzj0EAwMwNzEVMBMGA1UEChMMc2lnc3RvcmUuZGV2MR4wHAYDVQQDExVzaWdzdG9yZS1pbnRlcm1lZGlhdGUwHhcNMjYxMDE5MjIzMTE1WhcNMjYxMDE5MjI0MTE1WjAAMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE3edsuwnJ7jqqemLx4AZlgRX5HxhcWRE6INRFuuHbmZJClfx/ipjms6nRAaXEXcRASW3lKsIpYeSr/T1KEb00/6OCBdEwggXNMA4GA1UdDwEB/wQEAwIHgDATBgNVHSUEDDAKBggrBgEFBQcDAzAfBgNVHSMEGDAWgBTf0+nPViQRlvmo2OkoVaLGLhhkPzBrBgNVHREBAf8EYTBfhl1odHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkvLmdpdGh1Yi93b3JrZmxvd3MvYnVpbGQueWFtbEByZWZzL2hlYWRzL21haW4wOQYKKwYBBAGDvzABAQQraHR0cHM6Ly90b2tlbi5hY3Rpb25zLmdpdGh1YnVzZXJjb250ZW50LmNvbTAfBgorBgEEAYO/MAECBBF3b3JrZmxvd19kaXNwYXRjaDA2BgorBgEEAYO/MAEDBChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMBMGCisGAQQBg78wAQQEBUJ1aWxkMCsGCisGAQQBg78wAQUEHWh1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5MB0GCisGAQQBg78wAQYED3JlZnMvaGVhZHMvbWFpbjA7BgorBgEEAYO/MAEIBC0MK2h0dHBzOi8vdG9rZW4uYWN0aW9ucy5naXRodWJ1c2VyY29udGVudC5jb20wbQYKKwYBBAGDvzABCQRfDF1odHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkvLmdpdGh1Yi93b3JrZmxvd3MvYnVpbGQueWFtbEByZWZzL2hlYWRzL21haW4wOAYKKwYBBAGDvzABCgQqDChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMBsGCisGAQQBg78wAQsEDQwLc2VsZi1ob3N0ZWQwQAYKKwYBBAGDvzABDAQyDDBodHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkwOAYKKwYBBAGDvzABDQQqDChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMB8GCisGAQQBg78wAQ4EEQwPcmVmcy9oZWFkcy9tYWluMBoGCisGAQQBg78wAQ8EDAwKMTA3MTQ3NTUyOTAuBgorBgEEAYO/MAEQBCAMHmh0dHBzOi8vZ2l0aHViLmNvbS9odWdnaW5nZmFjZTAYBgorBgEEAYO/MAERBAoMCDI1NzIwNzQzMG0GCisGAQQBg78wARIEXwxdaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5Ly5naXRodWIvd29ya2Zsb3dzL2J1aWxkLnlhbWxAcmVmcy9oZWFkcy9tYWluMDgGCisGAQQBg78wARMEKgwoZmUyYmI2NmZlZTZjMmE3N2VjY2YwOGEwNTA4ZGM1NzQyOGQ0ODhlZDAhBgorBgEEAYO/MAEUBBMMEXdvcmtmbG93X2Rpc3BhdGNoMGQGCisGAQQBg78wARUEVgxUaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5L2FjdGlvbnMvcnVucy8zOTIzOTUxODEwNC9hdHRlbXB0cy8xMBYGCisGAQQBg78wARYECAwGcHVibGljMEYGCisGAQQBg78wARgEOAw2cmVwbzpodWdnaW5nZmFjZS9rZXJuZWxzLWNvbW11bml0eTpyZWY6cmVmcy9oZWFkcy9tYWluMIGKBgorBgEEAdZ5AgQCBHwEegB4AHYA3T0wasbHETJjGR4cmWc3AqJKXrjePK3/h4pygC8p7o4AAAGhVkpa6AAABAMARzBFAiEApQwCCX96bimcS6xRJKWJBsUiSoDvd+Pk/QYYFcktSVkCIEJ9/2rGZ8klXslytVkeKFQzr+e4l7NootT2Du8uVhacMAoGCCqGSM49BAMDA2cAMGQCMBAEOmqdi0D8R2V1qMg4/OyrvTJWu4oal0KEefWlu6GF7soePSHkA07ZyEiste4BDgIwf5dWNp7viPgDNSfubu6HszrI/4FgaMp5hPAHf5f//UA5nMbgENnYVvxWFKBCVZ8j"}, "tlogEntries":[{"logIndex":"3327211868", "logId":{"keyId":"wNI9atQGlz+VWfO6LRygH4QUfY/8W4RFwiT5i5WRgB0="}, "kindVersion":{"kind":"hashedrekord", "version":"0.0.1"}, "integratedTime":"1792449076", "inclusionPromise":{"signedEntryTimestamp":"MEUCIDMDUHqwsYRzaWixuHGJjWHFJ4/iRt+2UmGi39CfIy1dAiEA+wYw2O5mzlJ9y2j5cF2T4wEuFqjVZysBRi7ItBwq/98="}, "inclusionProof":{"logIndex":"3205307606", "rootHash":"0n+xNJnLMgp7sozEH6SRp+aDmxlicUgusBhZlZgsqJ0=", "treeSize":"3205307650", "hashes":["tfCB68vh29EGCuUdCAh0c7SkWTkhxdx4BgzG97ksQ6c=", "7mk/WY2lqGuRTbQKWtV6zQnOdC4m3It2CB/bnMVG0T0=", "cCW6sHzYYMHsBtt/nC0rcoi+UvaxxDmIhd7a2M452G4=", "dtWd+u22D+tZsDa84T5ZJW4w8w8UVj9lD0NJJDqjoHc=", "63fJ5uSg1REGSodPGe4zXFCUfdneQIGhUrpJayO0qkA=", "bFh/+uIRRy9QjkgdrzkEoxjpRf5jf0Ls3F1ULwcdbR4=", "M6d/KuQ//DRltSQtXzLZ1oBzSuFsf9G7CmHdK66xTD8=", "u+n9nq8zizYghdQXamNSsJ1yejTkpNxZoMaQxLGJgr8=", "w3goxeMWPeeGsKDqCTPfJBoJKCBga4F5WnerVtyidt8=", "0aSE/O6jCq82vSwoybvpblRnnP72+1gEInKy6Wyc3E0=", "5egUlZ4Pi0Ix/qLyCDXocsWywJCHqrde/S06vOZvlOM=", "QsXufUpoKtSjKAB19K1G00umQpWvPHlBNuRGFiiVjQQ=", "QIMkwQbwAewhLJd//5FFwI3SnyNQBGjCgJrelfG8vhc=", "Et17CACUv9QN64irUJuBjsrgRGkGnnJ9xacRV/aBdwQ=", "lqgEwkxhDMs/gdIAE4lmjfMwg7nDmv7lda6O0LGliCw=", "HovTPBWzoErvUg1u5sIi3x1ZonZ7yJekVsBrmpomCIA=", "G+nk7mgE5h6dUmv2UY7NDJrY3WvrZY31raaO1AETt+o=", "qlTWNDzERnTBQVcHHXXVPjiUP0SsikGxfcxVlgqVFDo=", "gg9GNApNke3+zNhl4tsWtmIycCFrOMCB3lys6XuSQNQ=", "NlOclldmKPA1tMKIskqHAwuQDYHp31YwpK+OOMrMccc=", "qxzHanAzz57SDdmJe0B7bJK72NTIBbwEMGBBKvDOROw=", "xH/DCseLHr9eKoYT8qsORZK7zVdEGYWHuVtsVrD95wY="], "checkpoint":{"envelope":"rekor.sigstore.dev - 1193050959916656506\n3205307650\n0n+xNJnLMgp7sozEH6SRp+aDmxlicUgusBhZlZgsqJ0=\n\n— rekor.sigstore.dev wNI9ajBFAiEA8G/cygCtz/9naiuZybe4zJFaOz4VLpZ1iK0GcNhfBekCIHsqXQhL6AgKCM9O4yJHjKODCbwaSWuacB9SlITFqmLl\n"}}, "canonicalizedBody":"eyJhcGlWZXJzaW9uIjoiMC4wLjEiLCJraW5kIjoiaGFzaGVkcmVrb3JkIiwic3BlYyI6eyJkYXRhIjp7Imhhc2giOnsiYWxnb3JpdGhtIjoic2hhMjU2IiwidmFsdWUiOiI0MDgzOGUyOGE4MjJkMmUzYTA1ZGM4Mjk2N2QzOTMzOWZiNTJlZTNiZjJiNWVkOGFkZWIyMmVjNTMzMzhmYjliIn19LCJzaWduYXR1cmUiOnsiY29udGVudCI6Ik1FUUNJQlE5WlNkbk9jOW5xZnQ4M0dETmE3ODlrZzBISFQ0YWxtUitFUDdoU2lVa0FpQk04NFl1MnJFNFY5NHZOY1pqUEJobWFWK2tBbzhjbFZEZmNlcmtyc1UwclE9PSIsInB1YmxpY0tleSI6eyJjb250ZW50IjoiTFMwdExTMUNSVWRKVGlCRFJWSlVTVVpKUTBGVVJTMHRMUzB0Q2sxSlNVaExla05EUW5KTFowRjNTVUpCWjBsVlRrTndTM2R4YTJvNVNFWTNMek5yZW5KdWFVbE9TMWQ2ZWxWTmQwTm5XVWxMYjFwSmVtb3dSVUYzVFhjS1RucEZWazFDVFVkQk1WVkZRMmhOVFdNeWJHNWpNMUoyWTIxVmRWcEhWakpOVWpSM1NFRlpSRlpSVVVSRmVGWjZZVmRrZW1SSE9YbGFVekZ3WW01U2JBcGpiVEZzV2tkc2FHUkhWWGRJYUdOT1RXcFplRTFFUlRWTmFrbDZUVlJGTVZkb1kwNU5hbGw0VFVSRk5VMXFTVEJOVkVVeFYycEJRVTFHYTNkRmQxbElDa3R2V2tsNmFqQkRRVkZaU1V0dldrbDZhakJFUVZGalJGRm5RVVV6WldSemRYZHVTamRxY1hGbGJVeDRORUZhYkdkU1dEVkllR2hqVjFKRk5rbE9Va1lLZFhWSVltMWFTa05zWm5ndmFYQnFiWE0yYmxKQllWaEZXR05TUVZOWE0yeExjMGx3V1dWVGNpOVVNVXRGWWpBd0x6WlBRMEprUlhkbloxaE9UVUUwUndwQk1WVmtSSGRGUWk5M1VVVkJkMGxJWjBSQlZFSm5UbFpJVTFWRlJFUkJTMEpuWjNKQ1owVkdRbEZqUkVGNlFXWkNaMDVXU0ZOTlJVZEVRVmRuUWxSbUNqQXJibEJXYVZGU2JIWnRiekpQYTI5V1lVeEhUR2hvYTFCNlFuSkNaMDVXU0ZKRlFrRm1PRVZaVkVKbWFHd3hiMlJJVW5kamVtOTJUREprY0dSSGFERUtXV2sxYW1JeU1IWmhTRlp1V2pKc2RWb3lXbWhaTWxWMllUSldlV0p0Vm5OamVURnFZakl4ZEdSWE5YQmtTR3QyVEcxa2NHUkhhREZaYVRrellqTktjZ3BhYlhoMlpETk5kbGx1Vm5CaVIxRjFaVmRHZEdKRlFubGFWMXA2VERKb2JGbFhVbnBNTWpGb1lWYzBkMDlSV1V0TGQxbENRa0ZIUkhaNlFVSkJVVkZ5Q21GSVVqQmpTRTAyVEhrNU1HSXlkR3hpYVRWb1dUTlNjR0l5TlhwTWJXUndaRWRvTVZsdVZucGFXRXBxWWpJMU1GcFhOVEJNYlU1MllsUkJaa0puYjNJS1FtZEZSVUZaVHk5TlFVVkRRa0pHTTJJelNuSmFiWGgyWkRFNWEyRllUbmRaV0ZKcVlVUkJNa0puYjNKQ1owVkZRVmxQTDAxQlJVUkNRMmh0V2xSS2FRcFphbGt5V20xV2JFNXRUWGxaVkdNeldsZE9hbHBxUVRSWlZFRXhUVVJvYTFsNlZUTk9SRWswV2tSUk5FOUhWbXROUWsxSFEybHpSMEZSVVVKbk56aDNDa0ZSVVVWQ1ZVb3hZVmQ0YTAxRGMwZERhWE5IUVZGUlFtYzNPSGRCVVZWRlNGZG9NVm95WkhCaWJXUnRXVmRPYkV3eWRHeGpiVFZzWWtoTmRGa3lPWFFLWWxoV2RXRllValZOUWpCSFEybHpSMEZSVVVKbk56aDNRVkZaUlVRelNteGFiazEyWVVkV2FGcElUWFppVjBad1ltcEJOMEpuYjNKQ1owVkZRVmxQTHdwTlFVVkpRa013VFVzeWFEQmtTRUo2VDJrNGRtUkhPWEphVnpSMVdWZE9NR0ZYT1hWamVUVnVZVmhTYjJSWFNqRmpNbFo1V1RJNWRXUkhWblZrUXpWcUNtSXlNSGRpVVZsTFMzZFpRa0pCUjBSMmVrRkNRMUZTWmtSR01XOWtTRkozWTNwdmRrd3laSEJrUjJneFdXazFhbUl5TUhaaFNGWnVXakpzZFZveVdtZ0tXVEpWZG1FeVZubGliVlp6WTNreGFtSXlNWFJrVnpWd1pFaHJka3h0WkhCa1IyZ3hXV2s1TTJJelNuSmFiWGgyWkROTmRsbHVWbkJpUjFGMVpWZEdkQXBpUlVKNVdsZGFla3d5YUd4WlYxSjZUREl4YUdGWE5IZFBRVmxMUzNkWlFrSkJSMFIyZWtGQ1EyZFJjVVJEYUcxYVZFcHBXV3BaTWxwdFZteE9iVTE1Q2xsVVl6TmFWMDVxV21wQk5GbFVRVEZOUkdocldYcFZNMDVFU1RSYVJGRTBUMGRXYTAxQ2MwZERhWE5IUVZGUlFtYzNPSGRCVVhORlJGRjNUR015Vm5NS1dta3hiMkl6VGpCYVYxRjNVVUZaUzB0M1dVSkNRVWRFZG5wQlFrUkJVWGxFUkVKdlpFaFNkMk42YjNaTU1tUndaRWRvTVZscE5XcGlNakIyWVVoV2JncGFNbXgxV2pKYWFGa3lWWFpoTWxaNVltMVdjMk41TVdwaU1qRjBaRmMxY0dSSWEzZFBRVmxMUzNkWlFrSkJSMFIyZWtGQ1JGRlJjVVJEYUcxYVZFcHBDbGxxV1RKYWJWWnNUbTFOZVZsVVl6TmFWMDVxV21wQk5GbFVRVEZOUkdocldYcFZNMDVFU1RSYVJGRTBUMGRXYTAxQ09FZERhWE5IUVZGUlFtYzNPSGNLUVZFMFJVVlJkMUJqYlZadFkzazViMXBYUm10amVUbDBXVmRzZFUxQ2IwZERhWE5IUVZGUlFtYzNPSGRCVVRoRlJFRjNTMDFVUVROTlZGRXpUbFJWZVFwUFZFRjFRbWR2Y2tKblJVVkJXVTh2VFVGRlVVSkRRVTFJYldnd1pFaENlazlwT0haYU1td3dZVWhXYVV4dFRuWmlVemx2WkZka2JtRlhOVzVhYlVacUNscFVRVmxDWjI5eVFtZEZSVUZaVHk5TlFVVlNRa0Z2VFVORVNURk9la2wzVG5wUmVrMUhNRWREYVhOSFFWRlJRbWMzT0hkQlVrbEZXSGQ0WkdGSVVqQUtZMGhOTmt4NU9XNWhXRkp2WkZkSmRWa3lPWFJNTW1neFdqSmtjR0p0WkcxWlYwNXNUREowYkdOdE5XeGlTRTEwV1RJNWRHSllWblZoV0ZJMVRIazFiZ3BoV0ZKdlpGZEpkbVF5T1hsaE1scHpZak5rZWt3eVNqRmhWM2hyVEc1c2FHSlhlRUZqYlZadFkzazViMXBYUm10amVUbDBXVmRzZFUxRVowZERhWE5IQ2tGUlVVSm5OemgzUVZKTlJVdG5kMjlhYlZWNVdXMUpNazV0V214YVZGcHFUVzFGTTA0eVZtcFpNbGwzVDBkRmQwNVVRVFJhUjAweFRucFJlVTlIVVRBS1QwUm9iRnBFUVdoQ1oyOXlRbWRGUlVGWlR5OU5RVVZWUWtKTlRVVllaSFpqYlhSdFlrYzVNMWd5VW5Cak0wSm9aRWRPYjAxSFVVZERhWE5IUVZGUlFncG5OemgzUVZKVlJWWm5lRlZoU0ZJd1kwaE5Oa3g1T1c1aFdGSnZaRmRKZFZreU9YUk1NbWd4V2pKa2NHSnRaRzFaVjA1c1RESjBiR050Tld4aVNFMTBDbGt5T1hSaVdGWjFZVmhTTlV3eVJtcGtSMngyWW01TmRtTnVWblZqZVRoNlQxUkplazlVVlhoUFJFVjNUa001YUdSSVVteGlXRUl3WTNrNGVFMUNXVWNLUTJselIwRlJVVUpuTnpoM1FWSlpSVU5CZDBkalNGWnBZa2RzYWsxRldVZERhWE5IUVZGUlFtYzNPSGRCVW1kRlQwRjNNbU50Vm5kaWVuQnZaRmRrYmdwaFZ6VnVXbTFHYWxwVE9YSmFXRXAxV2xkNGVreFhUblppVnpFeFltMXNNR1ZVY0hsYVYxazJZMjFXYldONU9XOWFWMFpyWTNrNWRGbFhiSFZOU1VkTENrSm5iM0pDWjBWRlFXUmFOVUZuVVVOQ1NIZEZaV2RDTkVGSVdVRXpWREIzWVhOaVNFVlVTbXBIVWpSamJWZGpNMEZ4U2t0WWNtcGxVRXN6TDJnMGNIa0taME00Y0Rkdk5FRkJRVWRvVm10d1lUWkJRVUZDUVUxQlVucENSa0ZwUlVGd1VYZERRMWc1Tm1KcGJXTlRObmhTU2t0WFNrSnpWV2xUYjBSMlpDdFFhd292VVZsWlJtTnJkRk5XYTBOSlJVbzVMekp5UjFvNGEyeFljMng1ZEZaclpVdEdVWHB5SzJVMGJEZE9iMjkwVkRKRWRUaDFWbWhoWTAxQmIwZERRM0ZIQ2xOTk5EbENRVTFFUVRKalFVMUhVVU5OUWtGRlQyMXhaR2t3UkRoU01sWXhjVTFuTkM5UGVYSjJWRXBYZFRSdllXd3dTMFZsWmxkc2RUWkhSamR6YjJVS1VGTklhMEV3TjFwNVJXbHpkR1UwUWtSblNYZG1OV1JYVG5BM2RtbFFaMFJPVTJaMVluVTJTSE42Y2trdk5FWm5ZVTF3TldoUVFVaG1OV1l2TDFWQk5RcHVUV0puUlU1dVdWWjJlRmRHUzBKRFZsbzRhZ290TFMwdExVVk9SQ0JEUlZKVVNVWkpRMEZVUlMwdExTMHRDZz09In19fX0="}], "timestampVerificationData":{"rfc3161Timestamps":[{"signedTimestamp":"MIICyzADAgEAMIICwgYJKoZIhvcNAQcCoIICszCCAq8CAQMxDTALBglghkgBZQMEAgEwgbgGCyqGSIb3DQEJEAEEoIGoBIGlMIGiAgEBBgkrBgEEAYO/MAIwMTANBglghkgBZQMEAgEFAAQgo8yY2H2W5TJzDMi71ksxVvtMI5Yk85L5ePrKyhWWYuMCFQCmdZ6tqY2ljRgQDYyeRbpJ+UjT0hgPMjAyNjEwMTkyMjMxMTZaMAMCAQGgMqQwMC4xFTATBgNVBAoTDHNpZ3N0b3JlLmRldjEVMBMGA1UEAxMMc2lnc3RvcmUtdHNhoAAxggHcMIIB2AIBATBRMDkxFTATBgNVBAoTDHNpZ3N0b3JlLmRldjEgMB4GA1UEAxMXc2lnc3RvcmUtdHNhLXNlbGZzaWduZWQCFDoTVC8MkGHuvMFDL8uKjosqI4sMMAsGCWCGSAFlAwQCAaCB/DAaBgkqhkiG9w0BCQMxDQYLKoZIhvcNAQkQAQQwHAYJKoZIhvcNAQkFMQ8XDTI2MTAxOTIyMzExNlowLwYJKoZIhvcNAQkEMSIEIOsIp7PBFnHyxWzK5qw8TodpDGtaKM4KYOySCt5HM4kaMIGOBgsqhkiG9w0BCRACLzF/MH0wezB5BCCF+Se8B6tiysO0Q1bBDvyBssaIP9p6uebYcNnROs0FtzBVMD2kOzA5MRUwEwYDVQQKEwxzaWdzdG9yZS5kZXYxIDAeBgNVBAMTF3NpZ3N0b3JlLXRzYS1zZWxmc2lnbmVkAhQ6E1QvDJBh7rzBQy/Lio6LKiOLDDAKBggqhkjOPQQDAgRoMGYCMQD799LG2rm5BHbJ+kOEhatzZmFa2atv0WZ5e8hV1h6NdYDUIQYkTYLtLfWUt9FdwEUCMQDd0xGZB1eu/B0kZhmEIO14nniSqN90/Zrh4CHX25WEB+okXRo3Ug41mIWRZeMv0bM="}]}}, "messageSignature":{"messageDigest":{"algorithm":"SHA2_256", "digest":"QIOOKKgi0uOgXcgpZ9OTOftS7jvyte2K3rIuxTM4+5s="}, "signature":"MEQCIBQ9ZSdnOc9nqft83GDNa789kg0HHT4almR+EP7hSiUkAiBM84Yu2rE4V94vNcZjPBhmaV+kAo8clVDfcerkrsU0rQ=="}} \ No newline at end of file diff --git a/build/torch212-rocm71-x86_64-linux/mha.py b/build/torch212-rocm71-x86_64-linux/mha.py new file mode 100644 index 0000000000000000000000000000000000000000..01e2d6a0fbee98d27129f2eb695ac38df73e6487 --- /dev/null +++ b/build/torch212-rocm71-x86_64-linux/mha.py @@ -0,0 +1,332 @@ +"""FlashAttention forward entry points backed by aiter's Composable-Kernel FMHA. + +This is the compiled (HIP / Composable Kernel) counterpart to the Triton +``aiter-flash-attn`` package. Only the *forward* pass is provided (dense and +variable-length), including support for learnable attention sinks via +``window_size[2]`` / ``s_aux`` (e.g. gpt-oss). The op set ships exclusively +the CK ``mha_fwd`` / ``mha_varlen_fwd`` kernels, so the sink path is always +taken (no fall-through to a non-sink ASM kernel). +""" + +from typing import Optional, Tuple, Union + +import torch +from torch import Tensor + +from ._ops import ops + + +def _round_multiple(x: int, m: int) -> int: + return (x + m - 1) // m * m + + +def _maybe_pad_headdim(t: Tensor, target: int) -> Tensor: + og = t.size(-1) + if og == target: + return t + return torch.nn.functional.pad(t, [0, target - og]) + + +def _unpack_window(window_size) -> Tuple[int, int, int]: + left = int(window_size[0]) + right = int(window_size[1]) + sink = int(window_size[2]) if len(window_size) == 3 else 0 + return left, right, sink + + +def _prep_sink(s_aux: Optional[Tensor], nheads: int, device) -> Optional[Tensor]: + if s_aux is None: + return None + assert s_aux.device == device, "s_aux must be on the same device as q" + assert s_aux.shape[0] == nheads, "s_aux must have shape [nheads_q]" + if s_aux.dtype != torch.float32: + s_aux = s_aux.to(torch.float32) + return s_aux + + +def _effective_sink_size(sink_size: int, s_aux: Optional[Tensor]) -> int: + # The CK kernel only takes the sink path when sink_size > 0. Callers using + # the transformers attention interface pass the per-head sink logits as + # ``s_aux`` but leave ``window_size`` a 2-tuple (sink_size == 0), so enable + # a single learnable sink automatically when s_aux is given. + if s_aux is not None and sink_size == 0: + return 1 + return sink_size + + +def flash_attn_func( + q: Tensor, + k: Tensor, + v: Tensor, + dropout_p: float = 0.0, + softmax_scale: Optional[float] = None, + causal: bool = False, + window_size: Tuple[int, ...] = (-1, -1, 0), + bias: Optional[Tensor] = None, + alibi_slopes: Optional[Tensor] = None, + return_lse: bool = False, + return_attn_probs: bool = False, + s_aux: Optional[Tensor] = None, + **kwargs, +) -> Union[Tensor, Tuple[Tensor, ...]]: + """Dense FlashAttention forward. + + ``q, k, v`` have shape ``(batch, seqlen, nheads, headdim)``. ``window_size`` + is ``(left, right, sink_size)``; ``s_aux`` is an optional ``(nheads_q,)`` + fp32 tensor of learnable sink logits. Extra keyword arguments (e.g. those + passed by the transformers flash-attention interface) are ignored. + """ + if softmax_scale is None: + softmax_scale = q.shape[-1] ** (-0.5) + window_size_left, window_size_right, sink_size = _unpack_window(window_size) + + nheads_q = q.size(2) + s_aux = _prep_sink(s_aux, nheads_q, q.device) + sink_size = _effective_sink_size(sink_size, s_aux) + + head_q_og = q.size(-1) + head_v_og = v.size(-1) + head_q = _round_multiple(head_q_og, 8) + head_v = _round_multiple(head_v_og, 8) + q = _maybe_pad_headdim(q, head_q) + k = _maybe_pad_headdim(k, head_q) + v = _maybe_pad_headdim(v, head_v) + + return_softmax = return_attn_probs and dropout_p > 0 + out, softmax_lse, S_dmask, _ = ops.mha_fwd( + q, + k, + v, + dropout_p, + softmax_scale, + causal, + window_size_left, + window_size_right, + sink_size, + return_lse or return_attn_probs, + return_softmax, + None, # cu_seqlens_q + None, # cu_seqlens_kv + None, # out + bias, + alibi_slopes, + None, # q_descale + None, # k_descale + None, # v_descale + s_aux, + None, # gen + ) + + out = out[..., :head_v_og] + if not (return_lse or return_attn_probs): + return out + result = (out,) + if return_lse: + result = result + (softmax_lse,) + if return_attn_probs: + result = result + (S_dmask,) + return result + + +def flash_attn_varlen_func( + q: Tensor, + k: Tensor, + v: Tensor, + cu_seqlens_q: Tensor, + cu_seqlens_k: Tensor, + max_seqlen_q: int, + max_seqlen_k: int, + dropout_p: float = 0.0, + softmax_scale: Optional[float] = None, + causal: bool = False, + window_size: Tuple[int, ...] = (-1, -1, 0), + bias: Optional[Tensor] = None, + alibi_slopes: Optional[Tensor] = None, + return_lse: bool = False, + return_attn_probs: bool = False, + block_table: Optional[Tensor] = None, + min_seqlen_q: int = 0, + logits_soft_cap: float = 0.0, + zero_tensors: bool = False, + cu_seqlens_q_padded: Optional[Tensor] = None, + cu_seqlens_k_padded: Optional[Tensor] = None, + s_aux: Optional[Tensor] = None, + **kwargs, +) -> Union[Tensor, Tuple[Tensor, ...]]: + """Variable-length / packed FlashAttention forward. + + ``q`` has shape ``(total_q, nheads, headdim)``; ``cu_seqlens_*`` are int32 + cumulative offsets of shape ``(batch + 1,)``. Sinks are supported through + ``window_size[2]`` / ``s_aux`` as in :func:`flash_attn_func`. Extra keyword + arguments (e.g. from the transformers flash-attention interface) are ignored. + """ + if softmax_scale is None: + softmax_scale = q.shape[-1] ** (-0.5) + window_size_left, window_size_right, sink_size = _unpack_window(window_size) + + nheads_q = q.size(1) + s_aux = _prep_sink(s_aux, nheads_q, q.device) + sink_size = _effective_sink_size(sink_size, s_aux) + + head_q_og = q.size(-1) + head_v_og = v.size(-1) + head_q = _round_multiple(head_q_og, 8) + head_v = _round_multiple(head_v_og, 8) + q = _maybe_pad_headdim(q, head_q) + k = _maybe_pad_headdim(k, head_q) + v = _maybe_pad_headdim(v, head_v) + + return_softmax = return_attn_probs and dropout_p > 0 + out, softmax_lse, S_dmask, _ = ops.mha_varlen_fwd( + q, + k, + v, + cu_seqlens_q, + cu_seqlens_k, + max_seqlen_q, + max_seqlen_k, + min_seqlen_q, + dropout_p, + softmax_scale, + logits_soft_cap, + zero_tensors, + causal, + window_size_left, + window_size_right, + sink_size, + return_lse or return_attn_probs, + return_softmax, + None, # out + block_table, + bias, + alibi_slopes, + None, # q_descale + None, # k_descale + None, # v_descale + None, # gen + cu_seqlens_q_padded, + cu_seqlens_k_padded, + s_aux, + ) + + out = out[..., :head_v_og] + if not (return_lse or return_attn_probs): + return out + result = (out,) + if return_lse: + result = result + (softmax_lse,) + if return_attn_probs: + result = result + (S_dmask,) + return result + + +def _append_to_contiguous_cache(k_cache, v_cache, k, v, cache_seqlens, Sq): + """In-place append of new k/v into a contiguous cache at per-batch offsets.""" + B = k_cache.shape[0] + if isinstance(cache_seqlens, int): + cache_seqlens = torch.full((B,), cache_seqlens, device=k_cache.device, dtype=torch.long) + pos = cache_seqlens.view(B, 1).to(torch.long) + torch.arange(Sq, device=k_cache.device).view(1, Sq) + bidx = torch.arange(B, device=k_cache.device).view(B, 1).expand(B, Sq) + k_cache[bidx, pos] = k.to(k_cache.dtype) + v_cache[bidx, pos] = v.to(v_cache.dtype) + + +def flash_attn_with_kvcache( + q: Tensor, + k_cache: Tensor, + v_cache: Tensor, + k: Optional[Tensor] = None, + v: Optional[Tensor] = None, + rotary_cos: Optional[Tensor] = None, + rotary_sin: Optional[Tensor] = None, + cache_seqlens=None, + block_table: Optional[Tensor] = None, + softmax_scale: Optional[float] = None, + causal: bool = True, + window_size: Tuple[int, ...] = (-1, -1, 0), + s_aux: Optional[Tensor] = None, + return_lse: bool = False, + **kwargs, +) -> Union[Tensor, Tuple[Tensor, ...]]: + """FlashAttention against a KV cache, built on the CK split-KV kernel. + + ``q`` is ``(batch, seqlen_q, nheads, headdim)``. For a contiguous cache, + ``k_cache``/``v_cache`` are ``(batch, seqlen_cache, nheads_k, headdim)`` and + ``cache_seqlens`` (int or ``(batch,)``) is the number of valid cache entries + *before* appending. If ``k``/``v`` (the new tokens) are given, they are + written into the cache in place. The query attends to the valid prefix + (``cache_seqlens`` + ``seqlen_q``) with bottom-right causal alignment. + + Notes / current limitations: + - Rotary embedding is not applied here; pass ``q``/``k`` already rotated + (``rotary_cos``/``rotary_sin`` are rejected). + - Paged caches are addressed via ``block_table`` (forwarded to the split-KV + varlen kernel); in-cache append for paged layouts must be done by the + caller (pass ``k=v=None``). + """ + if rotary_cos is not None or rotary_sin is not None: + raise NotImplementedError( + "flash_attn_with_kvcache does not apply rotary; rotate q/k beforehand." + ) + if softmax_scale is None: + softmax_scale = q.shape[-1] ** (-0.5) + window_size_left, window_size_right, sink_size = _unpack_window(window_size) + B, Sq, Hq, D = q.shape + s_aux = _prep_sink(s_aux, Hq, q.device) + sink_size = _effective_sink_size(sink_size, s_aux) + + paged = block_table is not None + + # ---- append new tokens into the cache (contiguous layout only) ---- + if k is not None and not paged: + assert cache_seqlens is not None, "cache_seqlens required to append into the cache" + _append_to_contiguous_cache(k_cache, v_cache, k, v, cache_seqlens, Sq) + elif k is not None and paged: + raise NotImplementedError( + "in-cache append for paged layouts is not supported; append externally " + "and call with k=v=None." + ) + + appended = 0 if k is None else Sq + + # ---- key lengths ---- + if cache_seqlens is None: + klens = torch.full((B,), k_cache.shape[1], device=q.device, dtype=torch.long) + elif isinstance(cache_seqlens, int): + klens = torch.full((B,), cache_seqlens + appended, device=q.device, dtype=torch.long) + else: + klens = cache_seqlens.to(q.device, torch.long) + appended + + uniform = bool((klens == klens[0]).all().item()) + + # ---- fast path: contiguous cache, uniform length -> dense kernel ---- + if not paged and uniform: + L = int(klens[0].item()) + return flash_attn_func( + q, k_cache[:, :L], v_cache[:, :L], + softmax_scale=softmax_scale, causal=causal, + window_size=(window_size_left, window_size_right, sink_size), + s_aux=s_aux, return_lse=return_lse, + ) + + # ---- general path: variable cache lengths -> split-KV varlen kernel ---- + cu_q = torch.arange(0, (B + 1) * Sq, Sq, device=q.device, dtype=torch.int32) + cu_k = torch.zeros(B + 1, device=q.device, dtype=torch.int32) + cu_k[1:] = torch.cumsum(klens, 0).to(torch.int32) + q_packed = q.reshape(B * Sq, Hq, D) + if paged: + k_packed, v_packed = k_cache, v_cache # kernel gathers via block_table + else: + k_packed = torch.cat([k_cache[b, : int(klens[b])] for b in range(B)], dim=0) + v_packed = torch.cat([v_cache[b, : int(klens[b])] for b in range(B)], dim=0) + + out = flash_attn_varlen_func( + q_packed, k_packed, v_packed, cu_q, cu_k, Sq, int(klens.max().item()), + softmax_scale=softmax_scale, causal=causal, + window_size=(window_size_left, window_size_right, sink_size), + block_table=block_table, s_aux=s_aux, return_lse=return_lse, + ) + if return_lse: + out, lse = out + return out.reshape(B, Sq, Hq, D), lse + return out.reshape(B, Sq, Hq, D) diff --git a/build/torch212-rocm71-x86_64-linux/symbols.json b/build/torch212-rocm71-x86_64-linux/symbols.json new file mode 100644 index 0000000000000000000000000000000000000000..e8348db4bd94712352e7fb3e83b1d94f47c2814b --- /dev/null +++ b/build/torch212-rocm71-x86_64-linux/symbols.json @@ -0,0 +1,349 @@ +{ + "schema_version": 1, + "module": "_aiter_flash_attn_ck_rocm_fe2bb66", + "functions": [ + { + "name": "flash_attn_func", + "qualname": "flash_attn_func", + "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha", + "kind": "function", + "doc": "Dense FlashAttention forward.\n\n``q, k, v`` have shape ``(batch, seqlen, nheads, headdim)``. ``window_size``\nis ``(left, right, sink_size)``; ``s_aux`` is an optional ``(nheads_q,)``\nfp32 tensor of learnable sink logits. Extra keyword arguments (e.g. those\npassed by the transformers flash-attention interface) are ignored.", + "signature": { + "parameters": [ + { + "name": "q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "v", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "dropout_p", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "0.0", + "annotation": "float" + }, + { + "name": "softmax_scale", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "float | None" + }, + { + "name": "causal", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "window_size", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "(-1, -1, 0)", + "annotation": "Tuple[int, ...]" + }, + { + "name": "bias", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "alibi_slopes", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "return_lse", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "return_attn_probs", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "s_aux", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "kwargs", + "kind": "VAR_KEYWORD", + "default": null, + "annotation": null + } + ], + "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]" + } + }, + { + "name": "flash_attn_varlen_func", + "qualname": "flash_attn_varlen_func", + "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha", + "kind": "function", + "doc": "Variable-length / packed FlashAttention forward.\n\n``q`` has shape ``(total_q, nheads, headdim)``; ``cu_seqlens_*`` are int32\ncumulative offsets of shape ``(batch + 1,)``. Sinks are supported through\n``window_size[2]`` / ``s_aux`` as in :func:`flash_attn_func`. Extra keyword\narguments (e.g. from the transformers flash-attention interface) are ignored.", + "signature": { + "parameters": [ + { + "name": "q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "v", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "cu_seqlens_q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "cu_seqlens_k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "max_seqlen_q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "int" + }, + { + "name": "max_seqlen_k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "int" + }, + { + "name": "dropout_p", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "0.0", + "annotation": "float" + }, + { + "name": "softmax_scale", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "float | None" + }, + { + "name": "causal", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "window_size", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "(-1, -1, 0)", + "annotation": "Tuple[int, ...]" + }, + { + "name": "bias", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "alibi_slopes", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "return_lse", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "return_attn_probs", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "block_table", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "min_seqlen_q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "0", + "annotation": "int" + }, + { + "name": "logits_soft_cap", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "0.0", + "annotation": "float" + }, + { + "name": "zero_tensors", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "cu_seqlens_q_padded", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "cu_seqlens_k_padded", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "s_aux", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "kwargs", + "kind": "VAR_KEYWORD", + "default": null, + "annotation": null + } + ], + "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]" + } + }, + { + "name": "flash_attn_with_kvcache", + "qualname": "flash_attn_with_kvcache", + "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha", + "kind": "function", + "doc": "FlashAttention against a KV cache, built on the CK split-KV kernel.\n\n``q`` is ``(batch, seqlen_q, nheads, headdim)``. For a contiguous cache,\n``k_cache``/``v_cache`` are ``(batch, seqlen_cache, nheads_k, headdim)`` and\n``cache_seqlens`` (int or ``(batch,)``) is the number of valid cache entries\n*before* appending. If ``k``/``v`` (the new tokens) are given, they are\nwritten into the cache in place. The query attends to the valid prefix\n(``cache_seqlens`` + ``seqlen_q``) with bottom-right causal alignment.\n\nNotes / current limitations:\n- Rotary embedding is not applied here; pass ``q``/``k`` already rotated\n (``rotary_cos``/``rotary_sin`` are rejected).\n- Paged caches are addressed via ``block_table`` (forwarded to the split-KV\n varlen kernel); in-cache append for paged layouts must be done by the\n caller (pass ``k=v=None``).", + "signature": { + "parameters": [ + { + "name": "q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "k_cache", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "v_cache", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "v", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "rotary_cos", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "rotary_sin", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "cache_seqlens", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": null + }, + { + "name": "block_table", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "softmax_scale", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "float | None" + }, + { + "name": "causal", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "True", + "annotation": "bool" + }, + { + "name": "window_size", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "(-1, -1, 0)", + "annotation": "Tuple[int, ...]" + }, + { + "name": "s_aux", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "return_lse", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "kwargs", + "kind": "VAR_KEYWORD", + "default": null, + "annotation": null + } + ], + "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]" + } + } + ], + "layers": [] +} diff --git a/build/torch212-rocm72-x86_64-linux/__init__.py b/build/torch212-rocm72-x86_64-linux/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..73acb58c1142a1ab2a6ed9d6f34bc6a53ab5e9b5 --- /dev/null +++ b/build/torch212-rocm72-x86_64-linux/__init__.py @@ -0,0 +1,27 @@ +"""AITER Composable-Kernel FlashAttention kernels for AMD ROCm (compiled HIP). + +Compiled (Composable Kernel / HIP) counterpart to the Triton-based +``aiter-flash-attn`` package, repackaged from the CK FMHA forward path of the +`ROCm/aiter `_ project as a self-contained +Hugging Face Hub kernel. Exposes the FlashAttention forward entry points with +learnable attention sink support (e.g. gpt-oss). +""" + +from .mha import ( + flash_attn_func, + flash_attn_varlen_func, + flash_attn_with_kvcache, +) + + +__kernel_metadata__ = { + "license": "mit", +} + + +__all__ = [ + "__kernel_metadata__", + "flash_attn_func", + "flash_attn_varlen_func", + "flash_attn_with_kvcache", +] diff --git a/build/torch212-rocm72-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so b/build/torch212-rocm72-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so new file mode 100644 index 0000000000000000000000000000000000000000..37a7f77767207e14a81b8ac4d7e8eea72b9aadde --- /dev/null +++ b/build/torch212-rocm72-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b2047a691f2945147456fef163603452d23707ec638cf2ea76fde4f596571ba +size 189896064 diff --git a/build/torch212-rocm72-x86_64-linux/_ops.py b/build/torch212-rocm72-x86_64-linux/_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..2b4dd56c220b61fd2c16e0c1a02ee91a491cef0c --- /dev/null +++ b/build/torch212-rocm72-x86_64-linux/_ops.py @@ -0,0 +1,9 @@ +import torch +from . import _aiter_flash_attn_ck_rocm_fe2bb66 +ops = torch.ops._aiter_flash_attn_ck_rocm_fe2bb66 + +def add_op_namespace_prefix(op_name: str): + """ + Prefix op by namespace. + """ + return f"_aiter_flash_attn_ck_rocm_fe2bb66::{op_name}" diff --git a/build/torch212-rocm72-x86_64-linux/metadata.json b/build/torch212-rocm72-x86_64-linux/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..a8506c1dd8f103524b5464d57c7bcdbf705a69aa --- /dev/null +++ b/build/torch212-rocm72-x86_64-linux/metadata.json @@ -0,0 +1,37 @@ +{ + "name": "aiter-flash-attn-ck", + "id": "_aiter_flash_attn_ck_rocm_fe2bb66", + "version": 1, + "kernels-minver": "0.14.0", + "license": "MIT", + "python-depends": [], + "kernel-depends": [], + "backend": { + "type": "rocm", + "archs": [ + "gfx942", + "gfx950" + ] + }, + "digest": { + "algorithm": "sha256", + "files": { + "__init__.py": "7wj1GTm2bqTLLCyT4xLuAk/+JWYrhINjY/ODMNWhgZ0=", + "_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so": "eyBHppHylFFHRW/vFjYDRS0jcH7GOM8up2/eT1llcbo=", + "_ops.py": "4WkdYDWWJp60zep+oW/5ym9vIeVMNfrgRCkP3o/w7yU=", + "mha.py": "zCAjY4O+DiCzmEL+W9sqXxZaqjsj4RJbWFcyPpUZejk=", + "symbols.json": "rhnF4H8Jsb6I+YPxIVFMgnLcTQJODEXbZ7ge77NZXAU=" + } + }, + "provenance": { + "kernel-builder": { + "version": "0.18.0-dev0", + "commit": "d054906a13710b811b96697b4635f03020842733", + "dirty": false + }, + "kernel": { + "commit": "fe2bb66fee6c2a77eccf08a0508dc57428d488ed", + "dirty": false + } + } +} \ No newline at end of file diff --git a/build/torch212-rocm72-x86_64-linux/metadata.json.sigstore b/build/torch212-rocm72-x86_64-linux/metadata.json.sigstore new file mode 100644 index 0000000000000000000000000000000000000000..e1ccc181d95c3dad10fba9caf94607ffc01c34bd --- /dev/null +++ b/build/torch212-rocm72-x86_64-linux/metadata.json.sigstore @@ -0,0 +1 @@ +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json", "verificationMaterial":{"certificate":{"rawBytes":"MIIHLTCCBrKgAwIBAgIUQ45I/pFC4gf04I3vc+3x0g7A174wCgYIKoZIzj0EAwMwNzEVMBMGA1UEChMMc2lnc3RvcmUuZGV2MR4wHAYDVQQDExVzaWdzdG9yZS1pbnRlcm1lZGlhdGUwHhcNMjYxMDE5MjIzMTE3WhcNMjYxMDE5MjI0MTE3WjAAMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEzjsfKDVfgfG/aQDPwqQuhwftoZEZK9S5hBiZEa1B80IEyXRgtQ/mvsOwoTYJ9mvxC1+nNSGJdimuEs+feeY6f6OCBdEwggXNMA4GA1UdDwEB/wQEAwIHgDATBgNVHSUEDDAKBggrBgEFBQcDAzAfBgNVHSMEGDAWgBTf0+nPViQRlvmo2OkoVaLGLhhkPzBrBgNVHREBAf8EYTBfhl1odHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkvLmdpdGh1Yi93b3JrZmxvd3MvYnVpbGQueWFtbEByZWZzL2hlYWRzL21haW4wOQYKKwYBBAGDvzABAQQraHR0cHM6Ly90b2tlbi5hY3Rpb25zLmdpdGh1YnVzZXJjb250ZW50LmNvbTAfBgorBgEEAYO/MAECBBF3b3JrZmxvd19kaXNwYXRjaDA2BgorBgEEAYO/MAEDBChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMBMGCisGAQQBg78wAQQEBUJ1aWxkMCsGCisGAQQBg78wAQUEHWh1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5MB0GCisGAQQBg78wAQYED3JlZnMvaGVhZHMvbWFpbjA7BgorBgEEAYO/MAEIBC0MK2h0dHBzOi8vdG9rZW4uYWN0aW9ucy5naXRodWJ1c2VyY29udGVudC5jb20wbQYKKwYBBAGDvzABCQRfDF1odHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkvLmdpdGh1Yi93b3JrZmxvd3MvYnVpbGQueWFtbEByZWZzL2hlYWRzL21haW4wOAYKKwYBBAGDvzABCgQqDChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMBsGCisGAQQBg78wAQsEDQwLc2VsZi1ob3N0ZWQwQAYKKwYBBAGDvzABDAQyDDBodHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkwOAYKKwYBBAGDvzABDQQqDChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMB8GCisGAQQBg78wAQ4EEQwPcmVmcy9oZWFkcy9tYWluMBoGCisGAQQBg78wAQ8EDAwKMTA3MTQ3NTUyOTAuBgorBgEEAYO/MAEQBCAMHmh0dHBzOi8vZ2l0aHViLmNvbS9odWdnaW5nZmFjZTAYBgorBgEEAYO/MAERBAoMCDI1NzIwNzQzMG0GCisGAQQBg78wARIEXwxdaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5Ly5naXRodWIvd29ya2Zsb3dzL2J1aWxkLnlhbWxAcmVmcy9oZWFkcy9tYWluMDgGCisGAQQBg78wARMEKgwoZmUyYmI2NmZlZTZjMmE3N2VjY2YwOGEwNTA4ZGM1NzQyOGQ0ODhlZDAhBgorBgEEAYO/MAEUBBMMEXdvcmtmbG93X2Rpc3BhdGNoMGQGCisGAQQBg78wARUEVgxUaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5L2FjdGlvbnMvcnVucy8zOTIzOTUxODEwNC9hdHRlbXB0cy8xMBYGCisGAQQBg78wARYECAwGcHVibGljMEYGCisGAQQBg78wARgEOAw2cmVwbzpodWdnaW5nZmFjZS9rZXJuZWxzLWNvbW11bml0eTpyZWY6cmVmcy9oZWFkcy9tYWluMIGKBgorBgEEAdZ5AgQCBHwEegB4AHYA3T0wasbHETJjGR4cmWc3AqJKXrjePK3/h4pygC8p7o4AAAGhVkphtgAABAMARzBFAiEA/V/XQ1zoAiy9Jx6LwsG2v9te8QMMdKgo5lxz6/CHUYYCIEBZ2id17T5aK+gexeVyI/uX1eFwmxl5wWBqdSKkHIfWMAoGCCqGSM49BAMDA2kAMGYCMQCtCXUfftdOhO4fxL+pXAyRkt9yANatw2Of1I2oXJa2N5+Ig+NVi5LLNvVkIwur1dMCMQDbzUgi38ARNQqw4NGCku4uStNr02qaF3jeDjSWPjcHfJfFxwDebjKCISt/W2u42Qo="}, "tlogEntries":[{"logIndex":"3327212104", "logId":{"keyId":"wNI9atQGlz+VWfO6LRygH4QUfY/8W4RFwiT5i5WRgB0="}, "kindVersion":{"kind":"hashedrekord", "version":"0.0.1"}, "integratedTime":"1792449077", "inclusionPromise":{"signedEntryTimestamp":"MEYCIQDRlFftrJStusZSVz3NRwN5GmWooDwyhs6XhSTx9F5qyAIhAImpjdodpPfG6f+FY+9Zu/ShY0j1Tdx6nmld19W2EEbc"}, "inclusionProof":{"logIndex":"3205307842", "rootHash":"Zzz8n5Mp9ft5puTfBQIBlCPo6wcO7myrHF6c/DCLCto=", "treeSize":"3205307848", "hashes":["PNwoDRxfezGRvsQGmM0Kf8ZjXCh17m6718g830ihWTY=", "4W/Uty2EfVeMffxMat+M956so0Ydun9CqlURs6q1cTk=", "ZecfiCPMv2u2lSRP0ST3fdJpMAgVpdtdk/HMrR51Tzg=", "RCHihIv1d3ARj25cx7a6BlBmIRIvbqbcYga+rFtfimU=", "NFzeBDxwmGphfWEGD4k9bLG9D0Y0y1ZYfIWGhkLVpVs=", "F4ofUVGe4nSzmh+WwnM4OaVmZh/WQERaH+u/wt8i8FM=", "0aSE/O6jCq82vSwoybvpblRnnP72+1gEInKy6Wyc3E0=", "5egUlZ4Pi0Ix/qLyCDXocsWywJCHqrde/S06vOZvlOM=", "QsXufUpoKtSjKAB19K1G00umQpWvPHlBNuRGFiiVjQQ=", "QIMkwQbwAewhLJd//5FFwI3SnyNQBGjCgJrelfG8vhc=", "Et17CACUv9QN64irUJuBjsrgRGkGnnJ9xacRV/aBdwQ=", "lqgEwkxhDMs/gdIAE4lmjfMwg7nDmv7lda6O0LGliCw=", "HovTPBWzoErvUg1u5sIi3x1ZonZ7yJekVsBrmpomCIA=", "G+nk7mgE5h6dUmv2UY7NDJrY3WvrZY31raaO1AETt+o=", "qlTWNDzERnTBQVcHHXXVPjiUP0SsikGxfcxVlgqVFDo=", "gg9GNApNke3+zNhl4tsWtmIycCFrOMCB3lys6XuSQNQ=", "NlOclldmKPA1tMKIskqHAwuQDYHp31YwpK+OOMrMccc=", "qxzHanAzz57SDdmJe0B7bJK72NTIBbwEMGBBKvDOROw=", "xH/DCseLHr9eKoYT8qsORZK7zVdEGYWHuVtsVrD95wY="], "checkpoint":{"envelope":"rekor.sigstore.dev - 1193050959916656506\n3205307848\nZzz8n5Mp9ft5puTfBQIBlCPo6wcO7myrHF6c/DCLCto=\n\n— rekor.sigstore.dev wNI9ajBEAiAp++VhzyYu83HrKvp+GRExuyer5bXN7+E3gZBsz5SdWwIgf1ZNOW/vRsfmirbx4AdVrRpcwLjtog7l6WYnLs3jdHI=\n"}}, "canonicalizedBody":"eyJhcGlWZXJzaW9uIjoiMC4wLjEiLCJraW5kIjoiaGFzaGVkcmVrb3JkIiwic3BlYyI6eyJkYXRhIjp7Imhhc2giOnsiYWxnb3JpdGhtIjoic2hhMjU2IiwidmFsdWUiOiIzNTcxOTkxYjg4ZDIyMGFkMmUwMjU2OWUyNmIxYjQzY2U5MjAxNjU0M2M1OGE3MWFjOWJiYjEzOTQxZDEwYzQzIn19LCJzaWduYXR1cmUiOnsiY29udGVudCI6Ik1FVUNJRE1QTHZ0UEp2QVZvQ0lrL3dkY1hWSW1NYUNCbW1aNHZ3c2FtN1hiZ0hhcUFpRUFwbTFwOTNZeWdSVHBOdExHeUhYZnFtSnE1MmdnbElYcURKeHY0TEhxMzdVPSIsInB1YmxpY0tleSI6eyJjb250ZW50IjoiTFMwdExTMUNSVWRKVGlCRFJWSlVTVVpKUTBGVVJTMHRMUzB0Q2sxSlNVaE1WRU5EUW5KTFowRjNTVUpCWjBsVlVUUTFTUzl3UmtNMFoyWXdORWt6ZG1Nck0zZ3daemRCTVRjMGQwTm5XVWxMYjFwSmVtb3dSVUYzVFhjS1RucEZWazFDVFVkQk1WVkZRMmhOVFdNeWJHNWpNMUoyWTIxVmRWcEhWakpOVWpSM1NFRlpSRlpSVVVSRmVGWjZZVmRrZW1SSE9YbGFVekZ3WW01U2JBcGpiVEZzV2tkc2FHUkhWWGRJYUdOT1RXcFplRTFFUlRWTmFrbDZUVlJGTTFkb1kwNU5hbGw0VFVSRk5VMXFTVEJOVkVVelYycEJRVTFHYTNkRmQxbElDa3R2V2tsNmFqQkRRVkZaU1V0dldrbDZhakJFUVZGalJGRm5RVVY2YW5ObVMwUldabWRtUnk5aFVVUlFkM0ZSZFdoM1puUnZXa1ZhU3psVE5XaENhVm9LUldFeFFqZ3dTVVY1V0ZKbmRGRXZiWFp6VDNkdlZGbEtPVzEyZUVNeEsyNU9VMGRLWkdsdGRVVnpLMlpsWlZrMlpqWlBRMEprUlhkbloxaE9UVUUwUndwQk1WVmtSSGRGUWk5M1VVVkJkMGxJWjBSQlZFSm5UbFpJVTFWRlJFUkJTMEpuWjNKQ1owVkdRbEZqUkVGNlFXWkNaMDVXU0ZOTlJVZEVRVmRuUWxSbUNqQXJibEJXYVZGU2JIWnRiekpQYTI5V1lVeEhUR2hvYTFCNlFuSkNaMDVXU0ZKRlFrRm1PRVZaVkVKbWFHd3hiMlJJVW5kamVtOTJUREprY0dSSGFERUtXV2sxYW1JeU1IWmhTRlp1V2pKc2RWb3lXbWhaTWxWMllUSldlV0p0Vm5OamVURnFZakl4ZEdSWE5YQmtTR3QyVEcxa2NHUkhhREZaYVRrellqTktjZ3BhYlhoMlpETk5kbGx1Vm5CaVIxRjFaVmRHZEdKRlFubGFWMXA2VERKb2JGbFhVbnBNTWpGb1lWYzBkMDlSV1V0TGQxbENRa0ZIUkhaNlFVSkJVVkZ5Q21GSVVqQmpTRTAyVEhrNU1HSXlkR3hpYVRWb1dUTlNjR0l5TlhwTWJXUndaRWRvTVZsdVZucGFXRXBxWWpJMU1GcFhOVEJNYlU1MllsUkJaa0puYjNJS1FtZEZSVUZaVHk5TlFVVkRRa0pHTTJJelNuSmFiWGgyWkRFNWEyRllUbmRaV0ZKcVlVUkJNa0puYjNKQ1owVkZRVmxQTDAxQlJVUkNRMmh0V2xSS2FRcFphbGt5V20xV2JFNXRUWGxaVkdNeldsZE9hbHBxUVRSWlZFRXhUVVJvYTFsNlZUTk9SRWswV2tSUk5FOUhWbXROUWsxSFEybHpSMEZSVVVKbk56aDNDa0ZSVVVWQ1ZVb3hZVmQ0YTAxRGMwZERhWE5IUVZGUlFtYzNPSGRCVVZWRlNGZG9NVm95WkhCaWJXUnRXVmRPYkV3eWRHeGpiVFZzWWtoTmRGa3lPWFFLWWxoV2RXRllValZOUWpCSFEybHpSMEZSVVVKbk56aDNRVkZaUlVRelNteGFiazEyWVVkV2FGcElUWFppVjBad1ltcEJOMEpuYjNKQ1owVkZRVmxQTHdwTlFVVkpRa013VFVzeWFEQmtTRUo2VDJrNGRtUkhPWEphVnpSMVdWZE9NR0ZYT1hWamVUVnVZVmhTYjJSWFNqRmpNbFo1V1RJNWRXUkhWblZrUXpWcUNtSXlNSGRpVVZsTFMzZFpRa0pCUjBSMmVrRkNRMUZTWmtSR01XOWtTRkozWTNwdmRrd3laSEJrUjJneFdXazFhbUl5TUhaaFNGWnVXakpzZFZveVdtZ0tXVEpWZG1FeVZubGliVlp6WTNreGFtSXlNWFJrVnpWd1pFaHJka3h0WkhCa1IyZ3hXV2s1TTJJelNuSmFiWGgyWkROTmRsbHVWbkJpUjFGMVpWZEdkQXBpUlVKNVdsZGFla3d5YUd4WlYxSjZUREl4YUdGWE5IZFBRVmxMUzNkWlFrSkJSMFIyZWtGQ1EyZFJjVVJEYUcxYVZFcHBXV3BaTWxwdFZteE9iVTE1Q2xsVVl6TmFWMDVxV21wQk5GbFVRVEZOUkdocldYcFZNMDVFU1RSYVJGRTBUMGRXYTAxQ2MwZERhWE5IUVZGUlFtYzNPSGRCVVhORlJGRjNUR015Vm5NS1dta3hiMkl6VGpCYVYxRjNVVUZaUzB0M1dVSkNRVWRFZG5wQlFrUkJVWGxFUkVKdlpFaFNkMk42YjNaTU1tUndaRWRvTVZscE5XcGlNakIyWVVoV2JncGFNbXgxV2pKYWFGa3lWWFpoTWxaNVltMVdjMk41TVdwaU1qRjBaRmMxY0dSSWEzZFBRVmxMUzNkWlFrSkJSMFIyZWtGQ1JGRlJjVVJEYUcxYVZFcHBDbGxxV1RKYWJWWnNUbTFOZVZsVVl6TmFWMDVxV21wQk5GbFVRVEZOUkdocldYcFZNMDVFU1RSYVJGRTBUMGRXYTAxQ09FZERhWE5IUVZGUlFtYzNPSGNLUVZFMFJVVlJkMUJqYlZadFkzazViMXBYUm10amVUbDBXVmRzZFUxQ2IwZERhWE5IUVZGUlFtYzNPSGRCVVRoRlJFRjNTMDFVUVROTlZGRXpUbFJWZVFwUFZFRjFRbWR2Y2tKblJVVkJXVTh2VFVGRlVVSkRRVTFJYldnd1pFaENlazlwT0haYU1td3dZVWhXYVV4dFRuWmlVemx2WkZka2JtRlhOVzVhYlVacUNscFVRVmxDWjI5eVFtZEZSVUZaVHk5TlFVVlNRa0Z2VFVORVNURk9la2wzVG5wUmVrMUhNRWREYVhOSFFWRlJRbWMzT0hkQlVrbEZXSGQ0WkdGSVVqQUtZMGhOTmt4NU9XNWhXRkp2WkZkSmRWa3lPWFJNTW1neFdqSmtjR0p0WkcxWlYwNXNUREowYkdOdE5XeGlTRTEwV1RJNWRHSllWblZoV0ZJMVRIazFiZ3BoV0ZKdlpGZEpkbVF5T1hsaE1scHpZak5rZWt3eVNqRmhWM2hyVEc1c2FHSlhlRUZqYlZadFkzazViMXBYUm10amVUbDBXVmRzZFUxRVowZERhWE5IQ2tGUlVVSm5OemgzUVZKTlJVdG5kMjlhYlZWNVdXMUpNazV0V214YVZGcHFUVzFGTTA0eVZtcFpNbGwzVDBkRmQwNVVRVFJhUjAweFRucFJlVTlIVVRBS1QwUm9iRnBFUVdoQ1oyOXlRbWRGUlVGWlR5OU5RVVZWUWtKTlRVVllaSFpqYlhSdFlrYzVNMWd5VW5Cak0wSm9aRWRPYjAxSFVVZERhWE5IUVZGUlFncG5OemgzUVZKVlJWWm5lRlZoU0ZJd1kwaE5Oa3g1T1c1aFdGSnZaRmRKZFZreU9YUk1NbWd4V2pKa2NHSnRaRzFaVjA1c1RESjBiR050Tld4aVNFMTBDbGt5T1hSaVdGWjFZVmhTTlV3eVJtcGtSMngyWW01TmRtTnVWblZqZVRoNlQxUkplazlVVlhoUFJFVjNUa001YUdSSVVteGlXRUl3WTNrNGVFMUNXVWNLUTJselIwRlJVVUpuTnpoM1FWSlpSVU5CZDBkalNGWnBZa2RzYWsxRldVZERhWE5IUVZGUlFtYzNPSGRCVW1kRlQwRjNNbU50Vm5kaWVuQnZaRmRrYmdwaFZ6VnVXbTFHYWxwVE9YSmFXRXAxV2xkNGVreFhUblppVnpFeFltMXNNR1ZVY0hsYVYxazJZMjFXYldONU9XOWFWMFpyWTNrNWRGbFhiSFZOU1VkTENrSm5iM0pDWjBWRlFXUmFOVUZuVVVOQ1NIZEZaV2RDTkVGSVdVRXpWREIzWVhOaVNFVlVTbXBIVWpSamJWZGpNMEZ4U2t0WWNtcGxVRXN6TDJnMGNIa0taME00Y0Rkdk5FRkJRVWRvVm10d2FIUm5RVUZDUVUxQlVucENSa0ZwUlVFdlZpOVlVVEY2YjBGcGVUbEtlRFpNZDNOSE1uWTVkR1U0VVUxTlpFdG5id28xYkhoNk5pOURTRlZaV1VOSlJVSmFNbWxrTVRkVU5XRkxLMmRsZUdWV2VVa3ZkVmd4WlVaM2JYaHNOWGRYUW5Ga1UwdHJTRWxtVjAxQmIwZERRM0ZIQ2xOTk5EbENRVTFFUVRKclFVMUhXVU5OVVVOMFExaFZabVowWkU5b1R6Um1lRXdyY0ZoQmVWSnJkRGw1UVU1aGRIY3lUMll4U1RKdldFcGhNazQxSzBrS1p5dE9WbWsxVEV4T2RsWnJTWGQxY2pGa1RVTk5VVVJpZWxWbmFUTTRRVkpPVVhGM05FNUhRMnQxTkhWVGRFNXlNREp4WVVZemFtVkVhbE5YVUdwalNBcG1TbVpHZUhkRVpXSnFTME5KVTNRdlZ6SjFOREpSYnowS0xTMHRMUzFGVGtRZ1EwVlNWRWxHU1VOQlZFVXRMUzB0TFFvPSJ9fX19"}], "timestampVerificationData":{"rfc3161Timestamps":[{"signedTimestamp":"MIICyTADAgEAMIICwAYJKoZIhvcNAQcCoIICsTCCAq0CAQMxDTALBglghkgBZQMEAgEwgbgGCyqGSIb3DQEJEAEEoIGoBIGlMIGiAgEBBgkrBgEEAYO/MAIwMTANBglghkgBZQMEAgEFAAQgmTkV835pgwxkvMNukL50F/XFs1cK4XlYhGVjIf6CtBsCFQCFgrdHkuvKvwpQj8RpZZmBpRIUlhgPMjAyNjEwMTkyMjMxMTdaMAMCAQGgMqQwMC4xFTATBgNVBAoTDHNpZ3N0b3JlLmRldjEVMBMGA1UEAxMMc2lnc3RvcmUtdHNhoAAxggHaMIIB1gIBATBRMDkxFTATBgNVBAoTDHNpZ3N0b3JlLmRldjEgMB4GA1UEAxMXc2lnc3RvcmUtdHNhLXNlbGZzaWduZWQCFDoTVC8MkGHuvMFDL8uKjosqI4sMMAsGCWCGSAFlAwQCAaCB/DAaBgkqhkiG9w0BCQMxDQYLKoZIhvcNAQkQAQQwHAYJKoZIhvcNAQkFMQ8XDTI2MTAxOTIyMzExN1owLwYJKoZIhvcNAQkEMSIEIJRd9HQUlnO59ivgTId0aXPDBwASdkAbCnMh269+NWAGMIGOBgsqhkiG9w0BCRACLzF/MH0wezB5BCCF+Se8B6tiysO0Q1bBDvyBssaIP9p6uebYcNnROs0FtzBVMD2kOzA5MRUwEwYDVQQKEwxzaWdzdG9yZS5kZXYxIDAeBgNVBAMTF3NpZ3N0b3JlLXRzYS1zZWxmc2lnbmVkAhQ6E1QvDJBh7rzBQy/Lio6LKiOLDDAKBggqhkjOPQQDAgRmMGQCMB8mlkizvZTubl6yOTamotJcCj+VOyU8JO51/iJzccSbUZEK7a58vYbacjQtKz0LdQIwPO96KwU6xzAQ+JfLYq0wy1m87aCJg68U22OyZ0sqwJvVatEW1hTkicd7Gh5GBX98"}]}}, "messageSignature":{"messageDigest":{"algorithm":"SHA2_256", "digest":"NXGZG4jSIK0uAlaeJrG0POkgFlQ8WKcaybuxOUHRDEM="}, "signature":"MEUCIDMPLvtPJvAVoCIk/wdcXVImMaCBmmZ4vwsam7XbgHaqAiEApm1p93YygRTpNtLGyHXfqmJq52gglIXqDJxv4LHq37U="}} \ No newline at end of file diff --git a/build/torch212-rocm72-x86_64-linux/mha.py b/build/torch212-rocm72-x86_64-linux/mha.py new file mode 100644 index 0000000000000000000000000000000000000000..01e2d6a0fbee98d27129f2eb695ac38df73e6487 --- /dev/null +++ b/build/torch212-rocm72-x86_64-linux/mha.py @@ -0,0 +1,332 @@ +"""FlashAttention forward entry points backed by aiter's Composable-Kernel FMHA. + +This is the compiled (HIP / Composable Kernel) counterpart to the Triton +``aiter-flash-attn`` package. Only the *forward* pass is provided (dense and +variable-length), including support for learnable attention sinks via +``window_size[2]`` / ``s_aux`` (e.g. gpt-oss). The op set ships exclusively +the CK ``mha_fwd`` / ``mha_varlen_fwd`` kernels, so the sink path is always +taken (no fall-through to a non-sink ASM kernel). +""" + +from typing import Optional, Tuple, Union + +import torch +from torch import Tensor + +from ._ops import ops + + +def _round_multiple(x: int, m: int) -> int: + return (x + m - 1) // m * m + + +def _maybe_pad_headdim(t: Tensor, target: int) -> Tensor: + og = t.size(-1) + if og == target: + return t + return torch.nn.functional.pad(t, [0, target - og]) + + +def _unpack_window(window_size) -> Tuple[int, int, int]: + left = int(window_size[0]) + right = int(window_size[1]) + sink = int(window_size[2]) if len(window_size) == 3 else 0 + return left, right, sink + + +def _prep_sink(s_aux: Optional[Tensor], nheads: int, device) -> Optional[Tensor]: + if s_aux is None: + return None + assert s_aux.device == device, "s_aux must be on the same device as q" + assert s_aux.shape[0] == nheads, "s_aux must have shape [nheads_q]" + if s_aux.dtype != torch.float32: + s_aux = s_aux.to(torch.float32) + return s_aux + + +def _effective_sink_size(sink_size: int, s_aux: Optional[Tensor]) -> int: + # The CK kernel only takes the sink path when sink_size > 0. Callers using + # the transformers attention interface pass the per-head sink logits as + # ``s_aux`` but leave ``window_size`` a 2-tuple (sink_size == 0), so enable + # a single learnable sink automatically when s_aux is given. + if s_aux is not None and sink_size == 0: + return 1 + return sink_size + + +def flash_attn_func( + q: Tensor, + k: Tensor, + v: Tensor, + dropout_p: float = 0.0, + softmax_scale: Optional[float] = None, + causal: bool = False, + window_size: Tuple[int, ...] = (-1, -1, 0), + bias: Optional[Tensor] = None, + alibi_slopes: Optional[Tensor] = None, + return_lse: bool = False, + return_attn_probs: bool = False, + s_aux: Optional[Tensor] = None, + **kwargs, +) -> Union[Tensor, Tuple[Tensor, ...]]: + """Dense FlashAttention forward. + + ``q, k, v`` have shape ``(batch, seqlen, nheads, headdim)``. ``window_size`` + is ``(left, right, sink_size)``; ``s_aux`` is an optional ``(nheads_q,)`` + fp32 tensor of learnable sink logits. Extra keyword arguments (e.g. those + passed by the transformers flash-attention interface) are ignored. + """ + if softmax_scale is None: + softmax_scale = q.shape[-1] ** (-0.5) + window_size_left, window_size_right, sink_size = _unpack_window(window_size) + + nheads_q = q.size(2) + s_aux = _prep_sink(s_aux, nheads_q, q.device) + sink_size = _effective_sink_size(sink_size, s_aux) + + head_q_og = q.size(-1) + head_v_og = v.size(-1) + head_q = _round_multiple(head_q_og, 8) + head_v = _round_multiple(head_v_og, 8) + q = _maybe_pad_headdim(q, head_q) + k = _maybe_pad_headdim(k, head_q) + v = _maybe_pad_headdim(v, head_v) + + return_softmax = return_attn_probs and dropout_p > 0 + out, softmax_lse, S_dmask, _ = ops.mha_fwd( + q, + k, + v, + dropout_p, + softmax_scale, + causal, + window_size_left, + window_size_right, + sink_size, + return_lse or return_attn_probs, + return_softmax, + None, # cu_seqlens_q + None, # cu_seqlens_kv + None, # out + bias, + alibi_slopes, + None, # q_descale + None, # k_descale + None, # v_descale + s_aux, + None, # gen + ) + + out = out[..., :head_v_og] + if not (return_lse or return_attn_probs): + return out + result = (out,) + if return_lse: + result = result + (softmax_lse,) + if return_attn_probs: + result = result + (S_dmask,) + return result + + +def flash_attn_varlen_func( + q: Tensor, + k: Tensor, + v: Tensor, + cu_seqlens_q: Tensor, + cu_seqlens_k: Tensor, + max_seqlen_q: int, + max_seqlen_k: int, + dropout_p: float = 0.0, + softmax_scale: Optional[float] = None, + causal: bool = False, + window_size: Tuple[int, ...] = (-1, -1, 0), + bias: Optional[Tensor] = None, + alibi_slopes: Optional[Tensor] = None, + return_lse: bool = False, + return_attn_probs: bool = False, + block_table: Optional[Tensor] = None, + min_seqlen_q: int = 0, + logits_soft_cap: float = 0.0, + zero_tensors: bool = False, + cu_seqlens_q_padded: Optional[Tensor] = None, + cu_seqlens_k_padded: Optional[Tensor] = None, + s_aux: Optional[Tensor] = None, + **kwargs, +) -> Union[Tensor, Tuple[Tensor, ...]]: + """Variable-length / packed FlashAttention forward. + + ``q`` has shape ``(total_q, nheads, headdim)``; ``cu_seqlens_*`` are int32 + cumulative offsets of shape ``(batch + 1,)``. Sinks are supported through + ``window_size[2]`` / ``s_aux`` as in :func:`flash_attn_func`. Extra keyword + arguments (e.g. from the transformers flash-attention interface) are ignored. + """ + if softmax_scale is None: + softmax_scale = q.shape[-1] ** (-0.5) + window_size_left, window_size_right, sink_size = _unpack_window(window_size) + + nheads_q = q.size(1) + s_aux = _prep_sink(s_aux, nheads_q, q.device) + sink_size = _effective_sink_size(sink_size, s_aux) + + head_q_og = q.size(-1) + head_v_og = v.size(-1) + head_q = _round_multiple(head_q_og, 8) + head_v = _round_multiple(head_v_og, 8) + q = _maybe_pad_headdim(q, head_q) + k = _maybe_pad_headdim(k, head_q) + v = _maybe_pad_headdim(v, head_v) + + return_softmax = return_attn_probs and dropout_p > 0 + out, softmax_lse, S_dmask, _ = ops.mha_varlen_fwd( + q, + k, + v, + cu_seqlens_q, + cu_seqlens_k, + max_seqlen_q, + max_seqlen_k, + min_seqlen_q, + dropout_p, + softmax_scale, + logits_soft_cap, + zero_tensors, + causal, + window_size_left, + window_size_right, + sink_size, + return_lse or return_attn_probs, + return_softmax, + None, # out + block_table, + bias, + alibi_slopes, + None, # q_descale + None, # k_descale + None, # v_descale + None, # gen + cu_seqlens_q_padded, + cu_seqlens_k_padded, + s_aux, + ) + + out = out[..., :head_v_og] + if not (return_lse or return_attn_probs): + return out + result = (out,) + if return_lse: + result = result + (softmax_lse,) + if return_attn_probs: + result = result + (S_dmask,) + return result + + +def _append_to_contiguous_cache(k_cache, v_cache, k, v, cache_seqlens, Sq): + """In-place append of new k/v into a contiguous cache at per-batch offsets.""" + B = k_cache.shape[0] + if isinstance(cache_seqlens, int): + cache_seqlens = torch.full((B,), cache_seqlens, device=k_cache.device, dtype=torch.long) + pos = cache_seqlens.view(B, 1).to(torch.long) + torch.arange(Sq, device=k_cache.device).view(1, Sq) + bidx = torch.arange(B, device=k_cache.device).view(B, 1).expand(B, Sq) + k_cache[bidx, pos] = k.to(k_cache.dtype) + v_cache[bidx, pos] = v.to(v_cache.dtype) + + +def flash_attn_with_kvcache( + q: Tensor, + k_cache: Tensor, + v_cache: Tensor, + k: Optional[Tensor] = None, + v: Optional[Tensor] = None, + rotary_cos: Optional[Tensor] = None, + rotary_sin: Optional[Tensor] = None, + cache_seqlens=None, + block_table: Optional[Tensor] = None, + softmax_scale: Optional[float] = None, + causal: bool = True, + window_size: Tuple[int, ...] = (-1, -1, 0), + s_aux: Optional[Tensor] = None, + return_lse: bool = False, + **kwargs, +) -> Union[Tensor, Tuple[Tensor, ...]]: + """FlashAttention against a KV cache, built on the CK split-KV kernel. + + ``q`` is ``(batch, seqlen_q, nheads, headdim)``. For a contiguous cache, + ``k_cache``/``v_cache`` are ``(batch, seqlen_cache, nheads_k, headdim)`` and + ``cache_seqlens`` (int or ``(batch,)``) is the number of valid cache entries + *before* appending. If ``k``/``v`` (the new tokens) are given, they are + written into the cache in place. The query attends to the valid prefix + (``cache_seqlens`` + ``seqlen_q``) with bottom-right causal alignment. + + Notes / current limitations: + - Rotary embedding is not applied here; pass ``q``/``k`` already rotated + (``rotary_cos``/``rotary_sin`` are rejected). + - Paged caches are addressed via ``block_table`` (forwarded to the split-KV + varlen kernel); in-cache append for paged layouts must be done by the + caller (pass ``k=v=None``). + """ + if rotary_cos is not None or rotary_sin is not None: + raise NotImplementedError( + "flash_attn_with_kvcache does not apply rotary; rotate q/k beforehand." + ) + if softmax_scale is None: + softmax_scale = q.shape[-1] ** (-0.5) + window_size_left, window_size_right, sink_size = _unpack_window(window_size) + B, Sq, Hq, D = q.shape + s_aux = _prep_sink(s_aux, Hq, q.device) + sink_size = _effective_sink_size(sink_size, s_aux) + + paged = block_table is not None + + # ---- append new tokens into the cache (contiguous layout only) ---- + if k is not None and not paged: + assert cache_seqlens is not None, "cache_seqlens required to append into the cache" + _append_to_contiguous_cache(k_cache, v_cache, k, v, cache_seqlens, Sq) + elif k is not None and paged: + raise NotImplementedError( + "in-cache append for paged layouts is not supported; append externally " + "and call with k=v=None." + ) + + appended = 0 if k is None else Sq + + # ---- key lengths ---- + if cache_seqlens is None: + klens = torch.full((B,), k_cache.shape[1], device=q.device, dtype=torch.long) + elif isinstance(cache_seqlens, int): + klens = torch.full((B,), cache_seqlens + appended, device=q.device, dtype=torch.long) + else: + klens = cache_seqlens.to(q.device, torch.long) + appended + + uniform = bool((klens == klens[0]).all().item()) + + # ---- fast path: contiguous cache, uniform length -> dense kernel ---- + if not paged and uniform: + L = int(klens[0].item()) + return flash_attn_func( + q, k_cache[:, :L], v_cache[:, :L], + softmax_scale=softmax_scale, causal=causal, + window_size=(window_size_left, window_size_right, sink_size), + s_aux=s_aux, return_lse=return_lse, + ) + + # ---- general path: variable cache lengths -> split-KV varlen kernel ---- + cu_q = torch.arange(0, (B + 1) * Sq, Sq, device=q.device, dtype=torch.int32) + cu_k = torch.zeros(B + 1, device=q.device, dtype=torch.int32) + cu_k[1:] = torch.cumsum(klens, 0).to(torch.int32) + q_packed = q.reshape(B * Sq, Hq, D) + if paged: + k_packed, v_packed = k_cache, v_cache # kernel gathers via block_table + else: + k_packed = torch.cat([k_cache[b, : int(klens[b])] for b in range(B)], dim=0) + v_packed = torch.cat([v_cache[b, : int(klens[b])] for b in range(B)], dim=0) + + out = flash_attn_varlen_func( + q_packed, k_packed, v_packed, cu_q, cu_k, Sq, int(klens.max().item()), + softmax_scale=softmax_scale, causal=causal, + window_size=(window_size_left, window_size_right, sink_size), + block_table=block_table, s_aux=s_aux, return_lse=return_lse, + ) + if return_lse: + out, lse = out + return out.reshape(B, Sq, Hq, D), lse + return out.reshape(B, Sq, Hq, D) diff --git a/build/torch212-rocm72-x86_64-linux/symbols.json b/build/torch212-rocm72-x86_64-linux/symbols.json new file mode 100644 index 0000000000000000000000000000000000000000..e8348db4bd94712352e7fb3e83b1d94f47c2814b --- /dev/null +++ b/build/torch212-rocm72-x86_64-linux/symbols.json @@ -0,0 +1,349 @@ +{ + "schema_version": 1, + "module": "_aiter_flash_attn_ck_rocm_fe2bb66", + "functions": [ + { + "name": "flash_attn_func", + "qualname": "flash_attn_func", + "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha", + "kind": "function", + "doc": "Dense FlashAttention forward.\n\n``q, k, v`` have shape ``(batch, seqlen, nheads, headdim)``. ``window_size``\nis ``(left, right, sink_size)``; ``s_aux`` is an optional ``(nheads_q,)``\nfp32 tensor of learnable sink logits. Extra keyword arguments (e.g. those\npassed by the transformers flash-attention interface) are ignored.", + "signature": { + "parameters": [ + { + "name": "q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "v", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "dropout_p", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "0.0", + "annotation": "float" + }, + { + "name": "softmax_scale", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "float | None" + }, + { + "name": "causal", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "window_size", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "(-1, -1, 0)", + "annotation": "Tuple[int, ...]" + }, + { + "name": "bias", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "alibi_slopes", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "return_lse", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "return_attn_probs", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "s_aux", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "kwargs", + "kind": "VAR_KEYWORD", + "default": null, + "annotation": null + } + ], + "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]" + } + }, + { + "name": "flash_attn_varlen_func", + "qualname": "flash_attn_varlen_func", + "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha", + "kind": "function", + "doc": "Variable-length / packed FlashAttention forward.\n\n``q`` has shape ``(total_q, nheads, headdim)``; ``cu_seqlens_*`` are int32\ncumulative offsets of shape ``(batch + 1,)``. Sinks are supported through\n``window_size[2]`` / ``s_aux`` as in :func:`flash_attn_func`. Extra keyword\narguments (e.g. from the transformers flash-attention interface) are ignored.", + "signature": { + "parameters": [ + { + "name": "q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "v", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "cu_seqlens_q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "cu_seqlens_k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "max_seqlen_q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "int" + }, + { + "name": "max_seqlen_k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "int" + }, + { + "name": "dropout_p", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "0.0", + "annotation": "float" + }, + { + "name": "softmax_scale", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "float | None" + }, + { + "name": "causal", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "window_size", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "(-1, -1, 0)", + "annotation": "Tuple[int, ...]" + }, + { + "name": "bias", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "alibi_slopes", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "return_lse", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "return_attn_probs", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "block_table", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "min_seqlen_q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "0", + "annotation": "int" + }, + { + "name": "logits_soft_cap", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "0.0", + "annotation": "float" + }, + { + "name": "zero_tensors", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "cu_seqlens_q_padded", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "cu_seqlens_k_padded", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "s_aux", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "kwargs", + "kind": "VAR_KEYWORD", + "default": null, + "annotation": null + } + ], + "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]" + } + }, + { + "name": "flash_attn_with_kvcache", + "qualname": "flash_attn_with_kvcache", + "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha", + "kind": "function", + "doc": "FlashAttention against a KV cache, built on the CK split-KV kernel.\n\n``q`` is ``(batch, seqlen_q, nheads, headdim)``. For a contiguous cache,\n``k_cache``/``v_cache`` are ``(batch, seqlen_cache, nheads_k, headdim)`` and\n``cache_seqlens`` (int or ``(batch,)``) is the number of valid cache entries\n*before* appending. If ``k``/``v`` (the new tokens) are given, they are\nwritten into the cache in place. The query attends to the valid prefix\n(``cache_seqlens`` + ``seqlen_q``) with bottom-right causal alignment.\n\nNotes / current limitations:\n- Rotary embedding is not applied here; pass ``q``/``k`` already rotated\n (``rotary_cos``/``rotary_sin`` are rejected).\n- Paged caches are addressed via ``block_table`` (forwarded to the split-KV\n varlen kernel); in-cache append for paged layouts must be done by the\n caller (pass ``k=v=None``).", + "signature": { + "parameters": [ + { + "name": "q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "k_cache", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "v_cache", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "v", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "rotary_cos", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "rotary_sin", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "cache_seqlens", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": null + }, + { + "name": "block_table", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "softmax_scale", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "float | None" + }, + { + "name": "causal", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "True", + "annotation": "bool" + }, + { + "name": "window_size", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "(-1, -1, 0)", + "annotation": "Tuple[int, ...]" + }, + { + "name": "s_aux", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "return_lse", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "kwargs", + "kind": "VAR_KEYWORD", + "default": null, + "annotation": null + } + ], + "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]" + } + } + ], + "layers": [] +} diff --git a/build/torch213-rocm71-x86_64-linux/__init__.py b/build/torch213-rocm71-x86_64-linux/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..73acb58c1142a1ab2a6ed9d6f34bc6a53ab5e9b5 --- /dev/null +++ b/build/torch213-rocm71-x86_64-linux/__init__.py @@ -0,0 +1,27 @@ +"""AITER Composable-Kernel FlashAttention kernels for AMD ROCm (compiled HIP). + +Compiled (Composable Kernel / HIP) counterpart to the Triton-based +``aiter-flash-attn`` package, repackaged from the CK FMHA forward path of the +`ROCm/aiter `_ project as a self-contained +Hugging Face Hub kernel. Exposes the FlashAttention forward entry points with +learnable attention sink support (e.g. gpt-oss). +""" + +from .mha import ( + flash_attn_func, + flash_attn_varlen_func, + flash_attn_with_kvcache, +) + + +__kernel_metadata__ = { + "license": "mit", +} + + +__all__ = [ + "__kernel_metadata__", + "flash_attn_func", + "flash_attn_varlen_func", + "flash_attn_with_kvcache", +] diff --git a/build/torch213-rocm71-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so b/build/torch213-rocm71-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so new file mode 100644 index 0000000000000000000000000000000000000000..8e9e544031f6e12b9b9a220efc7c4c01fed6340d --- /dev/null +++ b/build/torch213-rocm71-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a171f33a155682ab9672636dfb2f60fa988da30f3d8a9f6fff3d48fb6032ea6 +size 201962776 diff --git a/build/torch213-rocm71-x86_64-linux/_ops.py b/build/torch213-rocm71-x86_64-linux/_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..2b4dd56c220b61fd2c16e0c1a02ee91a491cef0c --- /dev/null +++ b/build/torch213-rocm71-x86_64-linux/_ops.py @@ -0,0 +1,9 @@ +import torch +from . import _aiter_flash_attn_ck_rocm_fe2bb66 +ops = torch.ops._aiter_flash_attn_ck_rocm_fe2bb66 + +def add_op_namespace_prefix(op_name: str): + """ + Prefix op by namespace. + """ + return f"_aiter_flash_attn_ck_rocm_fe2bb66::{op_name}" diff --git a/build/torch213-rocm71-x86_64-linux/metadata.json b/build/torch213-rocm71-x86_64-linux/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..72f5c91a3f2d6867dbc676bed931b007186257aa --- /dev/null +++ b/build/torch213-rocm71-x86_64-linux/metadata.json @@ -0,0 +1,37 @@ +{ + "name": "aiter-flash-attn-ck", + "id": "_aiter_flash_attn_ck_rocm_fe2bb66", + "version": 1, + "kernels-minver": "0.14.0", + "license": "MIT", + "python-depends": [], + "kernel-depends": [], + "backend": { + "type": "rocm", + "archs": [ + "gfx942", + "gfx950" + ] + }, + "digest": { + "algorithm": "sha256", + "files": { + "__init__.py": "7wj1GTm2bqTLLCyT4xLuAk/+JWYrhINjY/ODMNWhgZ0=", + "_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so": "KhcfM6FVaCq5ZyY237L2D6mI2jDz2Kn2//PUj7YDLqY=", + "_ops.py": "4WkdYDWWJp60zep+oW/5ym9vIeVMNfrgRCkP3o/w7yU=", + "mha.py": "zCAjY4O+DiCzmEL+W9sqXxZaqjsj4RJbWFcyPpUZejk=", + "symbols.json": "rhnF4H8Jsb6I+YPxIVFMgnLcTQJODEXbZ7ge77NZXAU=" + } + }, + "provenance": { + "kernel-builder": { + "version": "0.18.0-dev0", + "commit": "d054906a13710b811b96697b4635f03020842733", + "dirty": false + }, + "kernel": { + "commit": "fe2bb66fee6c2a77eccf08a0508dc57428d488ed", + "dirty": false + } + } +} \ No newline at end of file diff --git a/build/torch213-rocm71-x86_64-linux/metadata.json.sigstore b/build/torch213-rocm71-x86_64-linux/metadata.json.sigstore new file mode 100644 index 0000000000000000000000000000000000000000..12b5b9fae8e37417edf41d2aea55fcd9791046f0 --- /dev/null +++ b/build/torch213-rocm71-x86_64-linux/metadata.json.sigstore @@ -0,0 +1 @@ +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json", "verificationMaterial":{"certificate":{"rawBytes":"MIIHLTCCBrKgAwIBAgIUYhDBz3s6J742wscabPTQJuZF0E4wCgYIKoZIzj0EAwMwNzEVMBMGA1UEChMMc2lnc3RvcmUuZGV2MR4wHAYDVQQDExVzaWdzdG9yZS1pbnRlcm1lZGlhdGUwHhcNMjYxMDE5MjIzMTE4WhcNMjYxMDE5MjI0MTE4WjAAMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEp3g/dI77cg1tnq5eu54MWYrJ8cbb8SzEVubcy/U67+Aiwr4Y0MWc384cRXpKZj0FhdoGxrrETLe96k9d+zBzgaOCBdEwggXNMA4GA1UdDwEB/wQEAwIHgDATBgNVHSUEDDAKBggrBgEFBQcDAzAfBgNVHSMEGDAWgBTf0+nPViQRlvmo2OkoVaLGLhhkPzBrBgNVHREBAf8EYTBfhl1odHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkvLmdpdGh1Yi93b3JrZmxvd3MvYnVpbGQueWFtbEByZWZzL2hlYWRzL21haW4wOQYKKwYBBAGDvzABAQQraHR0cHM6Ly90b2tlbi5hY3Rpb25zLmdpdGh1YnVzZXJjb250ZW50LmNvbTAfBgorBgEEAYO/MAECBBF3b3JrZmxvd19kaXNwYXRjaDA2BgorBgEEAYO/MAEDBChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMBMGCisGAQQBg78wAQQEBUJ1aWxkMCsGCisGAQQBg78wAQUEHWh1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5MB0GCisGAQQBg78wAQYED3JlZnMvaGVhZHMvbWFpbjA7BgorBgEEAYO/MAEIBC0MK2h0dHBzOi8vdG9rZW4uYWN0aW9ucy5naXRodWJ1c2VyY29udGVudC5jb20wbQYKKwYBBAGDvzABCQRfDF1odHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkvLmdpdGh1Yi93b3JrZmxvd3MvYnVpbGQueWFtbEByZWZzL2hlYWRzL21haW4wOAYKKwYBBAGDvzABCgQqDChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMBsGCisGAQQBg78wAQsEDQwLc2VsZi1ob3N0ZWQwQAYKKwYBBAGDvzABDAQyDDBodHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkwOAYKKwYBBAGDvzABDQQqDChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMB8GCisGAQQBg78wAQ4EEQwPcmVmcy9oZWFkcy9tYWluMBoGCisGAQQBg78wAQ8EDAwKMTA3MTQ3NTUyOTAuBgorBgEEAYO/MAEQBCAMHmh0dHBzOi8vZ2l0aHViLmNvbS9odWdnaW5nZmFjZTAYBgorBgEEAYO/MAERBAoMCDI1NzIwNzQzMG0GCisGAQQBg78wARIEXwxdaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5Ly5naXRodWIvd29ya2Zsb3dzL2J1aWxkLnlhbWxAcmVmcy9oZWFkcy9tYWluMDgGCisGAQQBg78wARMEKgwoZmUyYmI2NmZlZTZjMmE3N2VjY2YwOGEwNTA4ZGM1NzQyOGQ0ODhlZDAhBgorBgEEAYO/MAEUBBMMEXdvcmtmbG93X2Rpc3BhdGNoMGQGCisGAQQBg78wARUEVgxUaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5L2FjdGlvbnMvcnVucy8zOTIzOTUxODEwNC9hdHRlbXB0cy8xMBYGCisGAQQBg78wARYECAwGcHVibGljMEYGCisGAQQBg78wARgEOAw2cmVwbzpodWdnaW5nZmFjZS9rZXJuZWxzLWNvbW11bml0eTpyZWY6cmVmcy9oZWFkcy9tYWluMIGKBgorBgEEAdZ5AgQCBHwEegB4AHYA3T0wasbHETJjGR4cmWc3AqJKXrjePK3/h4pygC8p7o4AAAGhVkplLwAABAMARzBFAiAYzcmarOFIpCdMHLsWMMcmiX3uiDqTtrSbhnugLl9KbQIhAJimKU6Gp6Muq/12zl25mMiXo7YzY9oPcsSkZyGlyJZwMAoGCCqGSM49BAMDA2kAMGYCMQD8NasCHiPCLAtolVO5/0rl5UfBymvjywi0NVKAJdcAvP61/IJlRbX5IZbatI+8YOsCMQCGhXN2BT6CTuaWC3qwLFzACyTV4uinQUnHOzEDc74z1+uGTmHfVTRDIi2/tFQuuNM="}, "tlogEntries":[{"logIndex":"3327212209", "logId":{"keyId":"wNI9atQGlz+VWfO6LRygH4QUfY/8W4RFwiT5i5WRgB0="}, "kindVersion":{"kind":"hashedrekord", "version":"0.0.1"}, "integratedTime":"1792449078", "inclusionPromise":{"signedEntryTimestamp":"MEQCIBsAQmhJQzBVzalYjG0DrFcgZBeNwlfSjSgAmWKG0sPLAiAg/J0Hzy6ImkWvWbDx3/jTT3+cRJDBm2PIrIDTQTwEuw=="}, "inclusionProof":{"logIndex":"3205307947", "rootHash":"W+fxNQGQRlQR8NP5epr4cduN8DmMRMB26mm1Ob1vTrs=", "treeSize":"3205307963", "hashes":["S7xwjCFRfoN65j+YWkYcaKfA6xnjEY+JNyw73H3LYfg=", "1ZB+altVOQMte65LrxRL5bTxYbBz43joc2eTBAALLpA=", "kBfrXfa6yvdyCvkB8KxtuAweraT6oqFH4L025R6tbjk=", "5AYjk/jc91WxLoeKt6AMwJV3depI3V6Yi1pPZMjQMJw=", "iFo9RWcriE709SPAUnZ3E60KJTBscfs+3bxe9WWL2OA=", "pe8st8CnUIruORVX9JaZ2kalqSjxb7oo2wfilP83+bk=", "gBGf/lDkk/5tDR8CLTuE5Pt2YTL1zjLs12T3vvxq7X4=", "0aSE/O6jCq82vSwoybvpblRnnP72+1gEInKy6Wyc3E0=", "5egUlZ4Pi0Ix/qLyCDXocsWywJCHqrde/S06vOZvlOM=", "QsXufUpoKtSjKAB19K1G00umQpWvPHlBNuRGFiiVjQQ=", "QIMkwQbwAewhLJd//5FFwI3SnyNQBGjCgJrelfG8vhc=", "Et17CACUv9QN64irUJuBjsrgRGkGnnJ9xacRV/aBdwQ=", "lqgEwkxhDMs/gdIAE4lmjfMwg7nDmv7lda6O0LGliCw=", "HovTPBWzoErvUg1u5sIi3x1ZonZ7yJekVsBrmpomCIA=", "G+nk7mgE5h6dUmv2UY7NDJrY3WvrZY31raaO1AETt+o=", "qlTWNDzERnTBQVcHHXXVPjiUP0SsikGxfcxVlgqVFDo=", "gg9GNApNke3+zNhl4tsWtmIycCFrOMCB3lys6XuSQNQ=", "NlOclldmKPA1tMKIskqHAwuQDYHp31YwpK+OOMrMccc=", "qxzHanAzz57SDdmJe0B7bJK72NTIBbwEMGBBKvDOROw=", "xH/DCseLHr9eKoYT8qsORZK7zVdEGYWHuVtsVrD95wY="], "checkpoint":{"envelope":"rekor.sigstore.dev - 1193050959916656506\n3205307963\nW+fxNQGQRlQR8NP5epr4cduN8DmMRMB26mm1Ob1vTrs=\n\n— rekor.sigstore.dev wNI9ajBFAiEAxuRsGa2QG9aIT9+2qqvRf3gfyS1DbDaXilVScnJohYUCIDNWE9xwg2HhHzFKbgG8xMK0LNtqkdJC9u2OLYFMDY6H\n"}}, "canonicalizedBody":"eyJhcGlWZXJzaW9uIjoiMC4wLjEiLCJraW5kIjoiaGFzaGVkcmVrb3JkIiwic3BlYyI6eyJkYXRhIjp7Imhhc2giOnsiYWxnb3JpdGhtIjoic2hhMjU2IiwidmFsdWUiOiIwMDhiYTkyYTNlY2M5MjA5ZDBiMDUxOTllZGEzY2IxYmE0ZWM4OGMzZTVmOGI0YWZhYzAwZmIyMTk4OTJkYjRjIn19LCJzaWduYXR1cmUiOnsiY29udGVudCI6Ik1FUUNIMXBXbzdsRVJEbXhFZlk3emZnMlBscjJTb2xqZWR1azZCZGhqYU1samZBQ0lRQzFaZks4Y3VFVERodW5rT0R0MTg3TTJwdlRpOEVWczg0WEwvNDY3RGtXbVE9PSIsInB1YmxpY0tleSI6eyJjb250ZW50IjoiTFMwdExTMUNSVWRKVGlCRFJWSlVTVVpKUTBGVVJTMHRMUzB0Q2sxSlNVaE1WRU5EUW5KTFowRjNTVUpCWjBsVldXaEVRbm96Y3paS056UXlkM05qWVdKUVZGRktkVnBHTUVVMGQwTm5XVWxMYjFwSmVtb3dSVUYzVFhjS1RucEZWazFDVFVkQk1WVkZRMmhOVFdNeWJHNWpNMUoyWTIxVmRWcEhWakpOVWpSM1NFRlpSRlpSVVVSRmVGWjZZVmRrZW1SSE9YbGFVekZ3WW01U2JBcGpiVEZzV2tkc2FHUkhWWGRJYUdOT1RXcFplRTFFUlRWTmFrbDZUVlJGTkZkb1kwNU5hbGw0VFVSRk5VMXFTVEJOVkVVMFYycEJRVTFHYTNkRmQxbElDa3R2V2tsNmFqQkRRVkZaU1V0dldrbDZhakJFUVZGalJGRm5RVVZ3TTJjdlpFazNOMk5uTVhSdWNUVmxkVFUwVFZkWmNrbzRZMkppT0ZONlJWWjFZbU1LZVM5Vk5qY3JRV2wzY2pSWk1FMVhZek00TkdOU1dIQkxXbW93Um1oa2IwZDRjbkpGVkV4bE9UWnJPV1FyZWtKNloyRlBRMEprUlhkbloxaE9UVUUwUndwQk1WVmtSSGRGUWk5M1VVVkJkMGxJWjBSQlZFSm5UbFpJVTFWRlJFUkJTMEpuWjNKQ1owVkdRbEZqUkVGNlFXWkNaMDVXU0ZOTlJVZEVRVmRuUWxSbUNqQXJibEJXYVZGU2JIWnRiekpQYTI5V1lVeEhUR2hvYTFCNlFuSkNaMDVXU0ZKRlFrRm1PRVZaVkVKbWFHd3hiMlJJVW5kamVtOTJUREprY0dSSGFERUtXV2sxYW1JeU1IWmhTRlp1V2pKc2RWb3lXbWhaTWxWMllUSldlV0p0Vm5OamVURnFZakl4ZEdSWE5YQmtTR3QyVEcxa2NHUkhhREZaYVRrellqTktjZ3BhYlhoMlpETk5kbGx1Vm5CaVIxRjFaVmRHZEdKRlFubGFWMXA2VERKb2JGbFhVbnBNTWpGb1lWYzBkMDlSV1V0TGQxbENRa0ZIUkhaNlFVSkJVVkZ5Q21GSVVqQmpTRTAyVEhrNU1HSXlkR3hpYVRWb1dUTlNjR0l5TlhwTWJXUndaRWRvTVZsdVZucGFXRXBxWWpJMU1GcFhOVEJNYlU1MllsUkJaa0puYjNJS1FtZEZSVUZaVHk5TlFVVkRRa0pHTTJJelNuSmFiWGgyWkRFNWEyRllUbmRaV0ZKcVlVUkJNa0puYjNKQ1owVkZRVmxQTDAxQlJVUkNRMmh0V2xSS2FRcFphbGt5V20xV2JFNXRUWGxaVkdNeldsZE9hbHBxUVRSWlZFRXhUVVJvYTFsNlZUTk9SRWswV2tSUk5FOUhWbXROUWsxSFEybHpSMEZSVVVKbk56aDNDa0ZSVVVWQ1ZVb3hZVmQ0YTAxRGMwZERhWE5IUVZGUlFtYzNPSGRCVVZWRlNGZG9NVm95WkhCaWJXUnRXVmRPYkV3eWRHeGpiVFZzWWtoTmRGa3lPWFFLWWxoV2RXRllValZOUWpCSFEybHpSMEZSVVVKbk56aDNRVkZaUlVRelNteGFiazEyWVVkV2FGcElUWFppVjBad1ltcEJOMEpuYjNKQ1owVkZRVmxQTHdwTlFVVkpRa013VFVzeWFEQmtTRUo2VDJrNGRtUkhPWEphVnpSMVdWZE9NR0ZYT1hWamVUVnVZVmhTYjJSWFNqRmpNbFo1V1RJNWRXUkhWblZrUXpWcUNtSXlNSGRpVVZsTFMzZFpRa0pCUjBSMmVrRkNRMUZTWmtSR01XOWtTRkozWTNwdmRrd3laSEJrUjJneFdXazFhbUl5TUhaaFNGWnVXakpzZFZveVdtZ0tXVEpWZG1FeVZubGliVlp6WTNreGFtSXlNWFJrVnpWd1pFaHJka3h0WkhCa1IyZ3hXV2s1TTJJelNuSmFiWGgyWkROTmRsbHVWbkJpUjFGMVpWZEdkQXBpUlVKNVdsZGFla3d5YUd4WlYxSjZUREl4YUdGWE5IZFBRVmxMUzNkWlFrSkJSMFIyZWtGQ1EyZFJjVVJEYUcxYVZFcHBXV3BaTWxwdFZteE9iVTE1Q2xsVVl6TmFWMDVxV21wQk5GbFVRVEZOUkdocldYcFZNMDVFU1RSYVJGRTBUMGRXYTAxQ2MwZERhWE5IUVZGUlFtYzNPSGRCVVhORlJGRjNUR015Vm5NS1dta3hiMkl6VGpCYVYxRjNVVUZaUzB0M1dVSkNRVWRFZG5wQlFrUkJVWGxFUkVKdlpFaFNkMk42YjNaTU1tUndaRWRvTVZscE5XcGlNakIyWVVoV2JncGFNbXgxV2pKYWFGa3lWWFpoTWxaNVltMVdjMk41TVdwaU1qRjBaRmMxY0dSSWEzZFBRVmxMUzNkWlFrSkJSMFIyZWtGQ1JGRlJjVVJEYUcxYVZFcHBDbGxxV1RKYWJWWnNUbTFOZVZsVVl6TmFWMDVxV21wQk5GbFVRVEZOUkdocldYcFZNMDVFU1RSYVJGRTBUMGRXYTAxQ09FZERhWE5IUVZGUlFtYzNPSGNLUVZFMFJVVlJkMUJqYlZadFkzazViMXBYUm10amVUbDBXVmRzZFUxQ2IwZERhWE5IUVZGUlFtYzNPSGRCVVRoRlJFRjNTMDFVUVROTlZGRXpUbFJWZVFwUFZFRjFRbWR2Y2tKblJVVkJXVTh2VFVGRlVVSkRRVTFJYldnd1pFaENlazlwT0haYU1td3dZVWhXYVV4dFRuWmlVemx2WkZka2JtRlhOVzVhYlVacUNscFVRVmxDWjI5eVFtZEZSVUZaVHk5TlFVVlNRa0Z2VFVORVNURk9la2wzVG5wUmVrMUhNRWREYVhOSFFWRlJRbWMzT0hkQlVrbEZXSGQ0WkdGSVVqQUtZMGhOTmt4NU9XNWhXRkp2WkZkSmRWa3lPWFJNTW1neFdqSmtjR0p0WkcxWlYwNXNUREowYkdOdE5XeGlTRTEwV1RJNWRHSllWblZoV0ZJMVRIazFiZ3BoV0ZKdlpGZEpkbVF5T1hsaE1scHpZak5rZWt3eVNqRmhWM2hyVEc1c2FHSlhlRUZqYlZadFkzazViMXBYUm10amVUbDBXVmRzZFUxRVowZERhWE5IQ2tGUlVVSm5OemgzUVZKTlJVdG5kMjlhYlZWNVdXMUpNazV0V214YVZGcHFUVzFGTTA0eVZtcFpNbGwzVDBkRmQwNVVRVFJhUjAweFRucFJlVTlIVVRBS1QwUm9iRnBFUVdoQ1oyOXlRbWRGUlVGWlR5OU5RVVZWUWtKTlRVVllaSFpqYlhSdFlrYzVNMWd5VW5Cak0wSm9aRWRPYjAxSFVVZERhWE5IUVZGUlFncG5OemgzUVZKVlJWWm5lRlZoU0ZJd1kwaE5Oa3g1T1c1aFdGSnZaRmRKZFZreU9YUk1NbWd4V2pKa2NHSnRaRzFaVjA1c1RESjBiR050Tld4aVNFMTBDbGt5T1hSaVdGWjFZVmhTTlV3eVJtcGtSMngyWW01TmRtTnVWblZqZVRoNlQxUkplazlVVlhoUFJFVjNUa001YUdSSVVteGlXRUl3WTNrNGVFMUNXVWNLUTJselIwRlJVVUpuTnpoM1FWSlpSVU5CZDBkalNGWnBZa2RzYWsxRldVZERhWE5IUVZGUlFtYzNPSGRCVW1kRlQwRjNNbU50Vm5kaWVuQnZaRmRrYmdwaFZ6VnVXbTFHYWxwVE9YSmFXRXAxV2xkNGVreFhUblppVnpFeFltMXNNR1ZVY0hsYVYxazJZMjFXYldONU9XOWFWMFpyWTNrNWRGbFhiSFZOU1VkTENrSm5iM0pDWjBWRlFXUmFOVUZuVVVOQ1NIZEZaV2RDTkVGSVdVRXpWREIzWVhOaVNFVlVTbXBIVWpSamJWZGpNMEZ4U2t0WWNtcGxVRXN6TDJnMGNIa0taME00Y0Rkdk5FRkJRVWRvVm10d2JFeDNRVUZDUVUxQlVucENSa0ZwUVZsNlkyMWhjazlHU1hCRFpFMUlUSE5YVFUxamJXbFlNM1ZwUkhGVWRISlRZZ3BvYm5WblRHdzVTMkpSU1doQlNtbHRTMVUyUjNBMlRYVnhMekV5ZW13eU5XMU5hVmh2TjFsNldUbHZVR056VTJ0YWVVZHNlVXBhZDAxQmIwZERRM0ZIQ2xOTk5EbENRVTFFUVRKclFVMUhXVU5OVVVRNFRtRnpRMGhwVUVOTVFYUnZiRlpQTlM4d2NtdzFWV1pDZVcxMmFubDNhVEJPVmt0QlNtUmpRWFpRTmpFS0wwbEtiRkppV0RWSldtSmhkRWtyT0ZsUGMwTk5VVU5IYUZoT01rSlVOa05VZFdGWFF6TnhkMHhHZWtGRGVWUldOSFZwYmxGVmJraFBla1ZFWXpjMGVnb3hLM1ZIVkcxSVpsWlVVa1JKYVRJdmRFWlJkWFZPVFQwS0xTMHRMUzFGVGtRZ1EwVlNWRWxHU1VOQlZFVXRMUzB0TFFvPSJ9fX19"}], "timestampVerificationData":{"rfc3161Timestamps":[{"signedTimestamp":"MIICyjADAgEAMIICwQYJKoZIhvcNAQcCoIICsjCCAq4CAQMxDTALBglghkgBZQMEAgEwgbcGCyqGSIb3DQEJEAEEoIGnBIGkMIGhAgEBBgkrBgEEAYO/MAIwMTANBglghkgBZQMEAgEFAAQgawL8w2Yhv+alAQlEIxeoeY+ZtrXWuMVUorIiTtT4TyICFGiL4yQnFEcyiQjCaUOyUI1utRZ8GA8yMDI2MTAxOTIyMzExOFowAwIBAaAypDAwLjEVMBMGA1UEChMMc2lnc3RvcmUuZGV2MRUwEwYDVQQDEwxzaWdzdG9yZS10c2GgADGCAdwwggHYAgEBMFEwOTEVMBMGA1UEChMMc2lnc3RvcmUuZGV2MSAwHgYDVQQDExdzaWdzdG9yZS10c2Etc2VsZnNpZ25lZAIUOhNULwyQYe68wUMvy4qOiyojiwwwCwYJYIZIAWUDBAIBoIH8MBoGCSqGSIb3DQEJAzENBgsqhkiG9w0BCRABBDAcBgkqhkiG9w0BCQUxDxcNMjYxMDE5MjIzMTE4WjAvBgkqhkiG9w0BCQQxIgQgKQVaP34iASGj9tDKvbngVn5AU9ONoeeIIpUzaQv1CrQwgY4GCyqGSIb3DQEJEAIvMX8wfTB7MHkEIIX5J7wHq2LKw7RDVsEO/IGyxog/2nq55thw2dE6zQW3MFUwPaQ7MDkxFTATBgNVBAoTDHNpZ3N0b3JlLmRldjEgMB4GA1UEAxMXc2lnc3RvcmUtdHNhLXNlbGZzaWduZWQCFDoTVC8MkGHuvMFDL8uKjosqI4sMMAoGCCqGSM49BAMCBGgwZgIxAOYdsdmqMP1jGpCvbUMDAMn9a7y2Yi0TW0qE804eLILU3GezzrxsPYzwIMjo22GsAAIxAKw3FmgrW1NuQOEFdtHwvQwje7S8o7tQedDvPWScBIyBcVGsh47i6D9JjfP8hsQAyA=="}]}}, "messageSignature":{"messageDigest":{"algorithm":"SHA2_256", "digest":"AIupKj7MkgnQsFGZ7aPLG6TsiMPl+LSvrAD7IZiS20w="}, "signature":"MEQCH1pWo7lERDmxEfY7zfg2Plr2Soljeduk6BdhjaMljfACIQC1ZfK8cuETDhunkODt187M2pvTi8EVs84XL/467DkWmQ=="}} \ No newline at end of file diff --git a/build/torch213-rocm71-x86_64-linux/mha.py b/build/torch213-rocm71-x86_64-linux/mha.py new file mode 100644 index 0000000000000000000000000000000000000000..01e2d6a0fbee98d27129f2eb695ac38df73e6487 --- /dev/null +++ b/build/torch213-rocm71-x86_64-linux/mha.py @@ -0,0 +1,332 @@ +"""FlashAttention forward entry points backed by aiter's Composable-Kernel FMHA. + +This is the compiled (HIP / Composable Kernel) counterpart to the Triton +``aiter-flash-attn`` package. Only the *forward* pass is provided (dense and +variable-length), including support for learnable attention sinks via +``window_size[2]`` / ``s_aux`` (e.g. gpt-oss). The op set ships exclusively +the CK ``mha_fwd`` / ``mha_varlen_fwd`` kernels, so the sink path is always +taken (no fall-through to a non-sink ASM kernel). +""" + +from typing import Optional, Tuple, Union + +import torch +from torch import Tensor + +from ._ops import ops + + +def _round_multiple(x: int, m: int) -> int: + return (x + m - 1) // m * m + + +def _maybe_pad_headdim(t: Tensor, target: int) -> Tensor: + og = t.size(-1) + if og == target: + return t + return torch.nn.functional.pad(t, [0, target - og]) + + +def _unpack_window(window_size) -> Tuple[int, int, int]: + left = int(window_size[0]) + right = int(window_size[1]) + sink = int(window_size[2]) if len(window_size) == 3 else 0 + return left, right, sink + + +def _prep_sink(s_aux: Optional[Tensor], nheads: int, device) -> Optional[Tensor]: + if s_aux is None: + return None + assert s_aux.device == device, "s_aux must be on the same device as q" + assert s_aux.shape[0] == nheads, "s_aux must have shape [nheads_q]" + if s_aux.dtype != torch.float32: + s_aux = s_aux.to(torch.float32) + return s_aux + + +def _effective_sink_size(sink_size: int, s_aux: Optional[Tensor]) -> int: + # The CK kernel only takes the sink path when sink_size > 0. Callers using + # the transformers attention interface pass the per-head sink logits as + # ``s_aux`` but leave ``window_size`` a 2-tuple (sink_size == 0), so enable + # a single learnable sink automatically when s_aux is given. + if s_aux is not None and sink_size == 0: + return 1 + return sink_size + + +def flash_attn_func( + q: Tensor, + k: Tensor, + v: Tensor, + dropout_p: float = 0.0, + softmax_scale: Optional[float] = None, + causal: bool = False, + window_size: Tuple[int, ...] = (-1, -1, 0), + bias: Optional[Tensor] = None, + alibi_slopes: Optional[Tensor] = None, + return_lse: bool = False, + return_attn_probs: bool = False, + s_aux: Optional[Tensor] = None, + **kwargs, +) -> Union[Tensor, Tuple[Tensor, ...]]: + """Dense FlashAttention forward. + + ``q, k, v`` have shape ``(batch, seqlen, nheads, headdim)``. ``window_size`` + is ``(left, right, sink_size)``; ``s_aux`` is an optional ``(nheads_q,)`` + fp32 tensor of learnable sink logits. Extra keyword arguments (e.g. those + passed by the transformers flash-attention interface) are ignored. + """ + if softmax_scale is None: + softmax_scale = q.shape[-1] ** (-0.5) + window_size_left, window_size_right, sink_size = _unpack_window(window_size) + + nheads_q = q.size(2) + s_aux = _prep_sink(s_aux, nheads_q, q.device) + sink_size = _effective_sink_size(sink_size, s_aux) + + head_q_og = q.size(-1) + head_v_og = v.size(-1) + head_q = _round_multiple(head_q_og, 8) + head_v = _round_multiple(head_v_og, 8) + q = _maybe_pad_headdim(q, head_q) + k = _maybe_pad_headdim(k, head_q) + v = _maybe_pad_headdim(v, head_v) + + return_softmax = return_attn_probs and dropout_p > 0 + out, softmax_lse, S_dmask, _ = ops.mha_fwd( + q, + k, + v, + dropout_p, + softmax_scale, + causal, + window_size_left, + window_size_right, + sink_size, + return_lse or return_attn_probs, + return_softmax, + None, # cu_seqlens_q + None, # cu_seqlens_kv + None, # out + bias, + alibi_slopes, + None, # q_descale + None, # k_descale + None, # v_descale + s_aux, + None, # gen + ) + + out = out[..., :head_v_og] + if not (return_lse or return_attn_probs): + return out + result = (out,) + if return_lse: + result = result + (softmax_lse,) + if return_attn_probs: + result = result + (S_dmask,) + return result + + +def flash_attn_varlen_func( + q: Tensor, + k: Tensor, + v: Tensor, + cu_seqlens_q: Tensor, + cu_seqlens_k: Tensor, + max_seqlen_q: int, + max_seqlen_k: int, + dropout_p: float = 0.0, + softmax_scale: Optional[float] = None, + causal: bool = False, + window_size: Tuple[int, ...] = (-1, -1, 0), + bias: Optional[Tensor] = None, + alibi_slopes: Optional[Tensor] = None, + return_lse: bool = False, + return_attn_probs: bool = False, + block_table: Optional[Tensor] = None, + min_seqlen_q: int = 0, + logits_soft_cap: float = 0.0, + zero_tensors: bool = False, + cu_seqlens_q_padded: Optional[Tensor] = None, + cu_seqlens_k_padded: Optional[Tensor] = None, + s_aux: Optional[Tensor] = None, + **kwargs, +) -> Union[Tensor, Tuple[Tensor, ...]]: + """Variable-length / packed FlashAttention forward. + + ``q`` has shape ``(total_q, nheads, headdim)``; ``cu_seqlens_*`` are int32 + cumulative offsets of shape ``(batch + 1,)``. Sinks are supported through + ``window_size[2]`` / ``s_aux`` as in :func:`flash_attn_func`. Extra keyword + arguments (e.g. from the transformers flash-attention interface) are ignored. + """ + if softmax_scale is None: + softmax_scale = q.shape[-1] ** (-0.5) + window_size_left, window_size_right, sink_size = _unpack_window(window_size) + + nheads_q = q.size(1) + s_aux = _prep_sink(s_aux, nheads_q, q.device) + sink_size = _effective_sink_size(sink_size, s_aux) + + head_q_og = q.size(-1) + head_v_og = v.size(-1) + head_q = _round_multiple(head_q_og, 8) + head_v = _round_multiple(head_v_og, 8) + q = _maybe_pad_headdim(q, head_q) + k = _maybe_pad_headdim(k, head_q) + v = _maybe_pad_headdim(v, head_v) + + return_softmax = return_attn_probs and dropout_p > 0 + out, softmax_lse, S_dmask, _ = ops.mha_varlen_fwd( + q, + k, + v, + cu_seqlens_q, + cu_seqlens_k, + max_seqlen_q, + max_seqlen_k, + min_seqlen_q, + dropout_p, + softmax_scale, + logits_soft_cap, + zero_tensors, + causal, + window_size_left, + window_size_right, + sink_size, + return_lse or return_attn_probs, + return_softmax, + None, # out + block_table, + bias, + alibi_slopes, + None, # q_descale + None, # k_descale + None, # v_descale + None, # gen + cu_seqlens_q_padded, + cu_seqlens_k_padded, + s_aux, + ) + + out = out[..., :head_v_og] + if not (return_lse or return_attn_probs): + return out + result = (out,) + if return_lse: + result = result + (softmax_lse,) + if return_attn_probs: + result = result + (S_dmask,) + return result + + +def _append_to_contiguous_cache(k_cache, v_cache, k, v, cache_seqlens, Sq): + """In-place append of new k/v into a contiguous cache at per-batch offsets.""" + B = k_cache.shape[0] + if isinstance(cache_seqlens, int): + cache_seqlens = torch.full((B,), cache_seqlens, device=k_cache.device, dtype=torch.long) + pos = cache_seqlens.view(B, 1).to(torch.long) + torch.arange(Sq, device=k_cache.device).view(1, Sq) + bidx = torch.arange(B, device=k_cache.device).view(B, 1).expand(B, Sq) + k_cache[bidx, pos] = k.to(k_cache.dtype) + v_cache[bidx, pos] = v.to(v_cache.dtype) + + +def flash_attn_with_kvcache( + q: Tensor, + k_cache: Tensor, + v_cache: Tensor, + k: Optional[Tensor] = None, + v: Optional[Tensor] = None, + rotary_cos: Optional[Tensor] = None, + rotary_sin: Optional[Tensor] = None, + cache_seqlens=None, + block_table: Optional[Tensor] = None, + softmax_scale: Optional[float] = None, + causal: bool = True, + window_size: Tuple[int, ...] = (-1, -1, 0), + s_aux: Optional[Tensor] = None, + return_lse: bool = False, + **kwargs, +) -> Union[Tensor, Tuple[Tensor, ...]]: + """FlashAttention against a KV cache, built on the CK split-KV kernel. + + ``q`` is ``(batch, seqlen_q, nheads, headdim)``. For a contiguous cache, + ``k_cache``/``v_cache`` are ``(batch, seqlen_cache, nheads_k, headdim)`` and + ``cache_seqlens`` (int or ``(batch,)``) is the number of valid cache entries + *before* appending. If ``k``/``v`` (the new tokens) are given, they are + written into the cache in place. The query attends to the valid prefix + (``cache_seqlens`` + ``seqlen_q``) with bottom-right causal alignment. + + Notes / current limitations: + - Rotary embedding is not applied here; pass ``q``/``k`` already rotated + (``rotary_cos``/``rotary_sin`` are rejected). + - Paged caches are addressed via ``block_table`` (forwarded to the split-KV + varlen kernel); in-cache append for paged layouts must be done by the + caller (pass ``k=v=None``). + """ + if rotary_cos is not None or rotary_sin is not None: + raise NotImplementedError( + "flash_attn_with_kvcache does not apply rotary; rotate q/k beforehand." + ) + if softmax_scale is None: + softmax_scale = q.shape[-1] ** (-0.5) + window_size_left, window_size_right, sink_size = _unpack_window(window_size) + B, Sq, Hq, D = q.shape + s_aux = _prep_sink(s_aux, Hq, q.device) + sink_size = _effective_sink_size(sink_size, s_aux) + + paged = block_table is not None + + # ---- append new tokens into the cache (contiguous layout only) ---- + if k is not None and not paged: + assert cache_seqlens is not None, "cache_seqlens required to append into the cache" + _append_to_contiguous_cache(k_cache, v_cache, k, v, cache_seqlens, Sq) + elif k is not None and paged: + raise NotImplementedError( + "in-cache append for paged layouts is not supported; append externally " + "and call with k=v=None." + ) + + appended = 0 if k is None else Sq + + # ---- key lengths ---- + if cache_seqlens is None: + klens = torch.full((B,), k_cache.shape[1], device=q.device, dtype=torch.long) + elif isinstance(cache_seqlens, int): + klens = torch.full((B,), cache_seqlens + appended, device=q.device, dtype=torch.long) + else: + klens = cache_seqlens.to(q.device, torch.long) + appended + + uniform = bool((klens == klens[0]).all().item()) + + # ---- fast path: contiguous cache, uniform length -> dense kernel ---- + if not paged and uniform: + L = int(klens[0].item()) + return flash_attn_func( + q, k_cache[:, :L], v_cache[:, :L], + softmax_scale=softmax_scale, causal=causal, + window_size=(window_size_left, window_size_right, sink_size), + s_aux=s_aux, return_lse=return_lse, + ) + + # ---- general path: variable cache lengths -> split-KV varlen kernel ---- + cu_q = torch.arange(0, (B + 1) * Sq, Sq, device=q.device, dtype=torch.int32) + cu_k = torch.zeros(B + 1, device=q.device, dtype=torch.int32) + cu_k[1:] = torch.cumsum(klens, 0).to(torch.int32) + q_packed = q.reshape(B * Sq, Hq, D) + if paged: + k_packed, v_packed = k_cache, v_cache # kernel gathers via block_table + else: + k_packed = torch.cat([k_cache[b, : int(klens[b])] for b in range(B)], dim=0) + v_packed = torch.cat([v_cache[b, : int(klens[b])] for b in range(B)], dim=0) + + out = flash_attn_varlen_func( + q_packed, k_packed, v_packed, cu_q, cu_k, Sq, int(klens.max().item()), + softmax_scale=softmax_scale, causal=causal, + window_size=(window_size_left, window_size_right, sink_size), + block_table=block_table, s_aux=s_aux, return_lse=return_lse, + ) + if return_lse: + out, lse = out + return out.reshape(B, Sq, Hq, D), lse + return out.reshape(B, Sq, Hq, D) diff --git a/build/torch213-rocm71-x86_64-linux/symbols.json b/build/torch213-rocm71-x86_64-linux/symbols.json new file mode 100644 index 0000000000000000000000000000000000000000..e8348db4bd94712352e7fb3e83b1d94f47c2814b --- /dev/null +++ b/build/torch213-rocm71-x86_64-linux/symbols.json @@ -0,0 +1,349 @@ +{ + "schema_version": 1, + "module": "_aiter_flash_attn_ck_rocm_fe2bb66", + "functions": [ + { + "name": "flash_attn_func", + "qualname": "flash_attn_func", + "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha", + "kind": "function", + "doc": "Dense FlashAttention forward.\n\n``q, k, v`` have shape ``(batch, seqlen, nheads, headdim)``. ``window_size``\nis ``(left, right, sink_size)``; ``s_aux`` is an optional ``(nheads_q,)``\nfp32 tensor of learnable sink logits. Extra keyword arguments (e.g. those\npassed by the transformers flash-attention interface) are ignored.", + "signature": { + "parameters": [ + { + "name": "q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "v", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "dropout_p", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "0.0", + "annotation": "float" + }, + { + "name": "softmax_scale", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "float | None" + }, + { + "name": "causal", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "window_size", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "(-1, -1, 0)", + "annotation": "Tuple[int, ...]" + }, + { + "name": "bias", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "alibi_slopes", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "return_lse", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "return_attn_probs", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "s_aux", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "kwargs", + "kind": "VAR_KEYWORD", + "default": null, + "annotation": null + } + ], + "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]" + } + }, + { + "name": "flash_attn_varlen_func", + "qualname": "flash_attn_varlen_func", + "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha", + "kind": "function", + "doc": "Variable-length / packed FlashAttention forward.\n\n``q`` has shape ``(total_q, nheads, headdim)``; ``cu_seqlens_*`` are int32\ncumulative offsets of shape ``(batch + 1,)``. Sinks are supported through\n``window_size[2]`` / ``s_aux`` as in :func:`flash_attn_func`. Extra keyword\narguments (e.g. from the transformers flash-attention interface) are ignored.", + "signature": { + "parameters": [ + { + "name": "q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "v", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "cu_seqlens_q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "cu_seqlens_k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "max_seqlen_q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "int" + }, + { + "name": "max_seqlen_k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "int" + }, + { + "name": "dropout_p", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "0.0", + "annotation": "float" + }, + { + "name": "softmax_scale", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "float | None" + }, + { + "name": "causal", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "window_size", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "(-1, -1, 0)", + "annotation": "Tuple[int, ...]" + }, + { + "name": "bias", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "alibi_slopes", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "return_lse", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "return_attn_probs", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "block_table", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "min_seqlen_q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "0", + "annotation": "int" + }, + { + "name": "logits_soft_cap", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "0.0", + "annotation": "float" + }, + { + "name": "zero_tensors", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "cu_seqlens_q_padded", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "cu_seqlens_k_padded", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "s_aux", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "kwargs", + "kind": "VAR_KEYWORD", + "default": null, + "annotation": null + } + ], + "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]" + } + }, + { + "name": "flash_attn_with_kvcache", + "qualname": "flash_attn_with_kvcache", + "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha", + "kind": "function", + "doc": "FlashAttention against a KV cache, built on the CK split-KV kernel.\n\n``q`` is ``(batch, seqlen_q, nheads, headdim)``. For a contiguous cache,\n``k_cache``/``v_cache`` are ``(batch, seqlen_cache, nheads_k, headdim)`` and\n``cache_seqlens`` (int or ``(batch,)``) is the number of valid cache entries\n*before* appending. If ``k``/``v`` (the new tokens) are given, they are\nwritten into the cache in place. The query attends to the valid prefix\n(``cache_seqlens`` + ``seqlen_q``) with bottom-right causal alignment.\n\nNotes / current limitations:\n- Rotary embedding is not applied here; pass ``q``/``k`` already rotated\n (``rotary_cos``/``rotary_sin`` are rejected).\n- Paged caches are addressed via ``block_table`` (forwarded to the split-KV\n varlen kernel); in-cache append for paged layouts must be done by the\n caller (pass ``k=v=None``).", + "signature": { + "parameters": [ + { + "name": "q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "k_cache", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "v_cache", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "v", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "rotary_cos", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "rotary_sin", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "cache_seqlens", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": null + }, + { + "name": "block_table", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "softmax_scale", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "float | None" + }, + { + "name": "causal", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "True", + "annotation": "bool" + }, + { + "name": "window_size", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "(-1, -1, 0)", + "annotation": "Tuple[int, ...]" + }, + { + "name": "s_aux", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "return_lse", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "kwargs", + "kind": "VAR_KEYWORD", + "default": null, + "annotation": null + } + ], + "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]" + } + } + ], + "layers": [] +} diff --git a/build/torch213-rocm72-x86_64-linux/__init__.py b/build/torch213-rocm72-x86_64-linux/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..73acb58c1142a1ab2a6ed9d6f34bc6a53ab5e9b5 --- /dev/null +++ b/build/torch213-rocm72-x86_64-linux/__init__.py @@ -0,0 +1,27 @@ +"""AITER Composable-Kernel FlashAttention kernels for AMD ROCm (compiled HIP). + +Compiled (Composable Kernel / HIP) counterpart to the Triton-based +``aiter-flash-attn`` package, repackaged from the CK FMHA forward path of the +`ROCm/aiter `_ project as a self-contained +Hugging Face Hub kernel. Exposes the FlashAttention forward entry points with +learnable attention sink support (e.g. gpt-oss). +""" + +from .mha import ( + flash_attn_func, + flash_attn_varlen_func, + flash_attn_with_kvcache, +) + + +__kernel_metadata__ = { + "license": "mit", +} + + +__all__ = [ + "__kernel_metadata__", + "flash_attn_func", + "flash_attn_varlen_func", + "flash_attn_with_kvcache", +] diff --git a/build/torch213-rocm72-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so b/build/torch213-rocm72-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so new file mode 100644 index 0000000000000000000000000000000000000000..b37372eaa45329eed1418947055b5b3e9408f663 --- /dev/null +++ b/build/torch213-rocm72-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:259c437d4b54465292cd91aebb1d16be39c0af5548fe1e8075b40cbf681ba362 +size 189896080 diff --git a/build/torch213-rocm72-x86_64-linux/_ops.py b/build/torch213-rocm72-x86_64-linux/_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..2b4dd56c220b61fd2c16e0c1a02ee91a491cef0c --- /dev/null +++ b/build/torch213-rocm72-x86_64-linux/_ops.py @@ -0,0 +1,9 @@ +import torch +from . import _aiter_flash_attn_ck_rocm_fe2bb66 +ops = torch.ops._aiter_flash_attn_ck_rocm_fe2bb66 + +def add_op_namespace_prefix(op_name: str): + """ + Prefix op by namespace. + """ + return f"_aiter_flash_attn_ck_rocm_fe2bb66::{op_name}" diff --git a/build/torch213-rocm72-x86_64-linux/metadata.json b/build/torch213-rocm72-x86_64-linux/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..4075babe5f7b615c60243a926d11294a73e844ff --- /dev/null +++ b/build/torch213-rocm72-x86_64-linux/metadata.json @@ -0,0 +1,37 @@ +{ + "name": "aiter-flash-attn-ck", + "id": "_aiter_flash_attn_ck_rocm_fe2bb66", + "version": 1, + "kernels-minver": "0.14.0", + "license": "MIT", + "python-depends": [], + "kernel-depends": [], + "backend": { + "type": "rocm", + "archs": [ + "gfx942", + "gfx950" + ] + }, + "digest": { + "algorithm": "sha256", + "files": { + "__init__.py": "7wj1GTm2bqTLLCyT4xLuAk/+JWYrhINjY/ODMNWhgZ0=", + "_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so": "JZxDfUtURlKSzZGuux0WvjnAr1VI/h6AdbQMv2gbo2I=", + "_ops.py": "4WkdYDWWJp60zep+oW/5ym9vIeVMNfrgRCkP3o/w7yU=", + "mha.py": "zCAjY4O+DiCzmEL+W9sqXxZaqjsj4RJbWFcyPpUZejk=", + "symbols.json": "rhnF4H8Jsb6I+YPxIVFMgnLcTQJODEXbZ7ge77NZXAU=" + } + }, + "provenance": { + "kernel-builder": { + "version": "0.18.0-dev0", + "commit": "d054906a13710b811b96697b4635f03020842733", + "dirty": false + }, + "kernel": { + "commit": "fe2bb66fee6c2a77eccf08a0508dc57428d488ed", + "dirty": false + } + } +} \ No newline at end of file diff --git a/build/torch213-rocm72-x86_64-linux/metadata.json.sigstore b/build/torch213-rocm72-x86_64-linux/metadata.json.sigstore new file mode 100644 index 0000000000000000000000000000000000000000..ff2259daa63bbd440c2d698107d62a09d461436b --- /dev/null +++ b/build/torch213-rocm72-x86_64-linux/metadata.json.sigstore @@ -0,0 +1 @@ +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json", "verificationMaterial":{"certificate":{"rawBytes":"MIIHLDCCBrKgAwIBAgIUAdU3pTwRJNYzZVFXQT+OotkQ09IwCgYIKoZIzj0EAwMwNzEVMBMGA1UEChMMc2lnc3RvcmUuZGV2MR4wHAYDVQQDExVzaWdzdG9yZS1pbnRlcm1lZGlhdGUwHhcNMjYxMDE5MjIzMTE5WhcNMjYxMDE5MjI0MTE5WjAAMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAENbz0sN6F1qS/uqaxlUpGRhrraK/FmjU5drFUDJtwVsx4jyAxVZcYtP5k4tOhLmvBUfQ8p0ZYOz0tBCRF8kwBEKOCBdEwggXNMA4GA1UdDwEB/wQEAwIHgDATBgNVHSUEDDAKBggrBgEFBQcDAzAfBgNVHSMEGDAWgBTf0+nPViQRlvmo2OkoVaLGLhhkPzBrBgNVHREBAf8EYTBfhl1odHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkvLmdpdGh1Yi93b3JrZmxvd3MvYnVpbGQueWFtbEByZWZzL2hlYWRzL21haW4wOQYKKwYBBAGDvzABAQQraHR0cHM6Ly90b2tlbi5hY3Rpb25zLmdpdGh1YnVzZXJjb250ZW50LmNvbTAfBgorBgEEAYO/MAECBBF3b3JrZmxvd19kaXNwYXRjaDA2BgorBgEEAYO/MAEDBChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMBMGCisGAQQBg78wAQQEBUJ1aWxkMCsGCisGAQQBg78wAQUEHWh1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5MB0GCisGAQQBg78wAQYED3JlZnMvaGVhZHMvbWFpbjA7BgorBgEEAYO/MAEIBC0MK2h0dHBzOi8vdG9rZW4uYWN0aW9ucy5naXRodWJ1c2VyY29udGVudC5jb20wbQYKKwYBBAGDvzABCQRfDF1odHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkvLmdpdGh1Yi93b3JrZmxvd3MvYnVpbGQueWFtbEByZWZzL2hlYWRzL21haW4wOAYKKwYBBAGDvzABCgQqDChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMBsGCisGAQQBg78wAQsEDQwLc2VsZi1ob3N0ZWQwQAYKKwYBBAGDvzABDAQyDDBodHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkwOAYKKwYBBAGDvzABDQQqDChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMB8GCisGAQQBg78wAQ4EEQwPcmVmcy9oZWFkcy9tYWluMBoGCisGAQQBg78wAQ8EDAwKMTA3MTQ3NTUyOTAuBgorBgEEAYO/MAEQBCAMHmh0dHBzOi8vZ2l0aHViLmNvbS9odWdnaW5nZmFjZTAYBgorBgEEAYO/MAERBAoMCDI1NzIwNzQzMG0GCisGAQQBg78wARIEXwxdaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5Ly5naXRodWIvd29ya2Zsb3dzL2J1aWxkLnlhbWxAcmVmcy9oZWFkcy9tYWluMDgGCisGAQQBg78wARMEKgwoZmUyYmI2NmZlZTZjMmE3N2VjY2YwOGEwNTA4ZGM1NzQyOGQ0ODhlZDAhBgorBgEEAYO/MAEUBBMMEXdvcmtmbG93X2Rpc3BhdGNoMGQGCisGAQQBg78wARUEVgxUaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5L2FjdGlvbnMvcnVucy8zOTIzOTUxODEwNC9hdHRlbXB0cy8xMBYGCisGAQQBg78wARYECAwGcHVibGljMEYGCisGAQQBg78wARgEOAw2cmVwbzpodWdnaW5nZmFjZS9rZXJuZWxzLWNvbW11bml0eTpyZWY6cmVmcy9oZWFkcy9tYWluMIGKBgorBgEEAdZ5AgQCBHwEegB4AHYA3T0wasbHETJjGR4cmWc3AqJKXrjePK3/h4pygC8p7o4AAAGhVkppNAAABAMARzBFAiBWBuR6FpOyLOOWMAY7Raa+y1zjfVxFDS8dyURJ0ss09QIhALcdG3oYTRFibGXJ7Wz9cyfkgvJh3SQTHKNEJjwBx5o/MAoGCCqGSM49BAMDA2gAMGUCMBAcqNAFQOvA/77WYckul8TuNP8QBz8/8HFmJHFVGW4EYUluasQLNPyU+lOzViZK6wIxAOpdD/a3I1lPxyp4BBjHE+nIh3BHHNr4U02nysHkPeiQ9/MNfMTHcpDWIj9PVqWVdw=="}, "tlogEntries":[{"logIndex":"3327212381", "logId":{"keyId":"wNI9atQGlz+VWfO6LRygH4QUfY/8W4RFwiT5i5WRgB0="}, "kindVersion":{"kind":"hashedrekord", "version":"0.0.1"}, "integratedTime":"1792449079", "inclusionPromise":{"signedEntryTimestamp":"MEYCIQDwIhVnM0p+I+1znPL7u/00bU0MOg4e8Hyhj3nnWZTt1wIhAIELhAMEVgwTO89LHgVwcLl64kgwc4NeV8SuCywQ1fKl"}, "inclusionProof":{"logIndex":"3205308119", "rootHash":"J18Ml/Hz33q/TkQ/+bGfFAfKaTpQ1ImC/2zdUiYa0bk=", "treeSize":"3205308120", "hashes":["FnhCnJ1NqbRvHdlskHk2Q9XphbrH/RpxVmBrfbnBtjw=", "M53i+aY3ihpxbBK5b1bxINWFZTi1cTqn73FM01EfQ0w=", "0+GvYyt6Ee0JzJgamd2DAOgx4meWgzd3KROe3GbKn7Q=", "A06up8Yj2JdoD4LP8LZfDtTp/KCaUL/iLjKkCuEXtBE=", "izH11NkX5DfACTAIVN+KFdWdZ0pwrSyRM0Q6z3mM9wc=", "ovTMeooH/ZoPRKl+pWmxUNW/Zp/d4rsHyC+jztr92m0=", "gBGf/lDkk/5tDR8CLTuE5Pt2YTL1zjLs12T3vvxq7X4=", "0aSE/O6jCq82vSwoybvpblRnnP72+1gEInKy6Wyc3E0=", "5egUlZ4Pi0Ix/qLyCDXocsWywJCHqrde/S06vOZvlOM=", "QsXufUpoKtSjKAB19K1G00umQpWvPHlBNuRGFiiVjQQ=", "QIMkwQbwAewhLJd//5FFwI3SnyNQBGjCgJrelfG8vhc=", "Et17CACUv9QN64irUJuBjsrgRGkGnnJ9xacRV/aBdwQ=", "lqgEwkxhDMs/gdIAE4lmjfMwg7nDmv7lda6O0LGliCw=", "HovTPBWzoErvUg1u5sIi3x1ZonZ7yJekVsBrmpomCIA=", "G+nk7mgE5h6dUmv2UY7NDJrY3WvrZY31raaO1AETt+o=", "qlTWNDzERnTBQVcHHXXVPjiUP0SsikGxfcxVlgqVFDo=", "gg9GNApNke3+zNhl4tsWtmIycCFrOMCB3lys6XuSQNQ=", "NlOclldmKPA1tMKIskqHAwuQDYHp31YwpK+OOMrMccc=", "qxzHanAzz57SDdmJe0B7bJK72NTIBbwEMGBBKvDOROw=", "xH/DCseLHr9eKoYT8qsORZK7zVdEGYWHuVtsVrD95wY="], "checkpoint":{"envelope":"rekor.sigstore.dev - 1193050959916656506\n3205308120\nJ18Ml/Hz33q/TkQ/+bGfFAfKaTpQ1ImC/2zdUiYa0bk=\n\n— rekor.sigstore.dev wNI9ajBEAiAJilSeua0jBtaifOcJjeOier6h36+wjsbRESJoTrCESQIgSU24+lff7Gxggo2SzFXOGXA1URgp985j+yKSc10dMcc=\n"}}, "canonicalizedBody":"eyJhcGlWZXJzaW9uIjoiMC4wLjEiLCJraW5kIjoiaGFzaGVkcmVrb3JkIiwic3BlYyI6eyJkYXRhIjp7Imhhc2giOnsiYWxnb3JpdGhtIjoic2hhMjU2IiwidmFsdWUiOiJlMGE5ZWI0ODY4Y2FiMzFkMzM3ODQwYmRkYjEwYTQxZTcxZTljZTlhMGI2YjJlYTViZmQ5NWY2YzhiYWMzNmI3In19LCJzaWduYXR1cmUiOnsiY29udGVudCI6Ik1FWUNJUUR3ekN5Wi9rQ0pJMjNDV0RSUmFrRHcwVEh6MmlXZk5TOFpGL1RjVThYWHFnSWhBSSt0MWNnNGhVNjFxY2hSbzFVaXVKL2tYbXc2bTVNS0s4ekg1WVVyaVJPMyIsInB1YmxpY0tleSI6eyJjb250ZW50IjoiTFMwdExTMUNSVWRKVGlCRFJWSlVTVVpKUTBGVVJTMHRMUzB0Q2sxSlNVaE1SRU5EUW5KTFowRjNTVUpCWjBsVlFXUlZNM0JVZDFKS1RsbDZXbFpHV0ZGVUswOXZkR3RSTURsSmQwTm5XVWxMYjFwSmVtb3dSVUYzVFhjS1RucEZWazFDVFVkQk1WVkZRMmhOVFdNeWJHNWpNMUoyWTIxVmRWcEhWakpOVWpSM1NFRlpSRlpSVVVSRmVGWjZZVmRrZW1SSE9YbGFVekZ3WW01U2JBcGpiVEZzV2tkc2FHUkhWWGRJYUdOT1RXcFplRTFFUlRWTmFrbDZUVlJGTlZkb1kwNU5hbGw0VFVSRk5VMXFTVEJOVkVVMVYycEJRVTFHYTNkRmQxbElDa3R2V2tsNmFqQkRRVkZaU1V0dldrbDZhakJFUVZGalJGRm5RVVZPWW5vd2MwNDJSakZ4VXk5MWNXRjRiRlZ3UjFKb2NuSmhTeTlHYldwVk5XUnlSbFVLUkVwMGQxWnplRFJxZVVGNFZscGpXWFJRTldzMGRFOW9URzEyUWxWbVVUaHdNRnBaVDNvd2RFSkRVa1k0YTNkQ1JVdFBRMEprUlhkbloxaE9UVUUwUndwQk1WVmtSSGRGUWk5M1VVVkJkMGxJWjBSQlZFSm5UbFpJVTFWRlJFUkJTMEpuWjNKQ1owVkdRbEZqUkVGNlFXWkNaMDVXU0ZOTlJVZEVRVmRuUWxSbUNqQXJibEJXYVZGU2JIWnRiekpQYTI5V1lVeEhUR2hvYTFCNlFuSkNaMDVXU0ZKRlFrRm1PRVZaVkVKbWFHd3hiMlJJVW5kamVtOTJUREprY0dSSGFERUtXV2sxYW1JeU1IWmhTRlp1V2pKc2RWb3lXbWhaTWxWMllUSldlV0p0Vm5OamVURnFZakl4ZEdSWE5YQmtTR3QyVEcxa2NHUkhhREZaYVRrellqTktjZ3BhYlhoMlpETk5kbGx1Vm5CaVIxRjFaVmRHZEdKRlFubGFWMXA2VERKb2JGbFhVbnBNTWpGb1lWYzBkMDlSV1V0TGQxbENRa0ZIUkhaNlFVSkJVVkZ5Q21GSVVqQmpTRTAyVEhrNU1HSXlkR3hpYVRWb1dUTlNjR0l5TlhwTWJXUndaRWRvTVZsdVZucGFXRXBxWWpJMU1GcFhOVEJNYlU1MllsUkJaa0puYjNJS1FtZEZSVUZaVHk5TlFVVkRRa0pHTTJJelNuSmFiWGgyWkRFNWEyRllUbmRaV0ZKcVlVUkJNa0puYjNKQ1owVkZRVmxQTDAxQlJVUkNRMmh0V2xSS2FRcFphbGt5V20xV2JFNXRUWGxaVkdNeldsZE9hbHBxUVRSWlZFRXhUVVJvYTFsNlZUTk9SRWswV2tSUk5FOUhWbXROUWsxSFEybHpSMEZSVVVKbk56aDNDa0ZSVVVWQ1ZVb3hZVmQ0YTAxRGMwZERhWE5IUVZGUlFtYzNPSGRCVVZWRlNGZG9NVm95WkhCaWJXUnRXVmRPYkV3eWRHeGpiVFZzWWtoTmRGa3lPWFFLWWxoV2RXRllValZOUWpCSFEybHpSMEZSVVVKbk56aDNRVkZaUlVRelNteGFiazEyWVVkV2FGcElUWFppVjBad1ltcEJOMEpuYjNKQ1owVkZRVmxQTHdwTlFVVkpRa013VFVzeWFEQmtTRUo2VDJrNGRtUkhPWEphVnpSMVdWZE9NR0ZYT1hWamVUVnVZVmhTYjJSWFNqRmpNbFo1V1RJNWRXUkhWblZrUXpWcUNtSXlNSGRpVVZsTFMzZFpRa0pCUjBSMmVrRkNRMUZTWmtSR01XOWtTRkozWTNwdmRrd3laSEJrUjJneFdXazFhbUl5TUhaaFNGWnVXakpzZFZveVdtZ0tXVEpWZG1FeVZubGliVlp6WTNreGFtSXlNWFJrVnpWd1pFaHJka3h0WkhCa1IyZ3hXV2s1TTJJelNuSmFiWGgyWkROTmRsbHVWbkJpUjFGMVpWZEdkQXBpUlVKNVdsZGFla3d5YUd4WlYxSjZUREl4YUdGWE5IZFBRVmxMUzNkWlFrSkJSMFIyZWtGQ1EyZFJjVVJEYUcxYVZFcHBXV3BaTWxwdFZteE9iVTE1Q2xsVVl6TmFWMDVxV21wQk5GbFVRVEZOUkdocldYcFZNMDVFU1RSYVJGRTBUMGRXYTAxQ2MwZERhWE5IUVZGUlFtYzNPSGRCVVhORlJGRjNUR015Vm5NS1dta3hiMkl6VGpCYVYxRjNVVUZaUzB0M1dVSkNRVWRFZG5wQlFrUkJVWGxFUkVKdlpFaFNkMk42YjNaTU1tUndaRWRvTVZscE5XcGlNakIyWVVoV2JncGFNbXgxV2pKYWFGa3lWWFpoTWxaNVltMVdjMk41TVdwaU1qRjBaRmMxY0dSSWEzZFBRVmxMUzNkWlFrSkJSMFIyZWtGQ1JGRlJjVVJEYUcxYVZFcHBDbGxxV1RKYWJWWnNUbTFOZVZsVVl6TmFWMDVxV21wQk5GbFVRVEZOUkdocldYcFZNMDVFU1RSYVJGRTBUMGRXYTAxQ09FZERhWE5IUVZGUlFtYzNPSGNLUVZFMFJVVlJkMUJqYlZadFkzazViMXBYUm10amVUbDBXVmRzZFUxQ2IwZERhWE5IUVZGUlFtYzNPSGRCVVRoRlJFRjNTMDFVUVROTlZGRXpUbFJWZVFwUFZFRjFRbWR2Y2tKblJVVkJXVTh2VFVGRlVVSkRRVTFJYldnd1pFaENlazlwT0haYU1td3dZVWhXYVV4dFRuWmlVemx2WkZka2JtRlhOVzVhYlVacUNscFVRVmxDWjI5eVFtZEZSVUZaVHk5TlFVVlNRa0Z2VFVORVNURk9la2wzVG5wUmVrMUhNRWREYVhOSFFWRlJRbWMzT0hkQlVrbEZXSGQ0WkdGSVVqQUtZMGhOTmt4NU9XNWhXRkp2WkZkSmRWa3lPWFJNTW1neFdqSmtjR0p0WkcxWlYwNXNUREowYkdOdE5XeGlTRTEwV1RJNWRHSllWblZoV0ZJMVRIazFiZ3BoV0ZKdlpGZEpkbVF5T1hsaE1scHpZak5rZWt3eVNqRmhWM2hyVEc1c2FHSlhlRUZqYlZadFkzazViMXBYUm10amVUbDBXVmRzZFUxRVowZERhWE5IQ2tGUlVVSm5OemgzUVZKTlJVdG5kMjlhYlZWNVdXMUpNazV0V214YVZGcHFUVzFGTTA0eVZtcFpNbGwzVDBkRmQwNVVRVFJhUjAweFRucFJlVTlIVVRBS1QwUm9iRnBFUVdoQ1oyOXlRbWRGUlVGWlR5OU5RVVZWUWtKTlRVVllaSFpqYlhSdFlrYzVNMWd5VW5Cak0wSm9aRWRPYjAxSFVVZERhWE5IUVZGUlFncG5OemgzUVZKVlJWWm5lRlZoU0ZJd1kwaE5Oa3g1T1c1aFdGSnZaRmRKZFZreU9YUk1NbWd4V2pKa2NHSnRaRzFaVjA1c1RESjBiR050Tld4aVNFMTBDbGt5T1hSaVdGWjFZVmhTTlV3eVJtcGtSMngyWW01TmRtTnVWblZqZVRoNlQxUkplazlVVlhoUFJFVjNUa001YUdSSVVteGlXRUl3WTNrNGVFMUNXVWNLUTJselIwRlJVVUpuTnpoM1FWSlpSVU5CZDBkalNGWnBZa2RzYWsxRldVZERhWE5IUVZGUlFtYzNPSGRCVW1kRlQwRjNNbU50Vm5kaWVuQnZaRmRrYmdwaFZ6VnVXbTFHYWxwVE9YSmFXRXAxV2xkNGVreFhUblppVnpFeFltMXNNR1ZVY0hsYVYxazJZMjFXYldONU9XOWFWMFpyWTNrNWRGbFhiSFZOU1VkTENrSm5iM0pDWjBWRlFXUmFOVUZuVVVOQ1NIZEZaV2RDTkVGSVdVRXpWREIzWVhOaVNFVlVTbXBIVWpSamJWZGpNMEZ4U2t0WWNtcGxVRXN6TDJnMGNIa0taME00Y0Rkdk5FRkJRVWRvVm10d2NFNUJRVUZDUVUxQlVucENSa0ZwUWxkQ2RWSTJSbkJQZVV4UFQxZE5RVmszVW1GaEsza3hlbXBtVm5oR1JGTTRaQXA1VlZKS01ITnpNRGxSU1doQlRHTmtSek52V1ZSU1JtbGlSMWhLTjFkNk9XTjVabXRuZGtwb00xTlJWRWhMVGtWS2FuZENlRFZ2TDAxQmIwZERRM0ZIQ2xOTk5EbENRVTFFUVRKblFVMUhWVU5OUWtGamNVNUJSbEZQZGtFdk56ZFhXV05yZFd3NFZIVk9VRGhSUW5vNEx6aElSbTFLU0VaV1IxYzBSVmxWYkhVS1lYTlJURTVRZVZVcmJFOTZWbWxhU3paM1NYaEJUM0JrUkM5aE0wa3hiRkI0ZVhBMFFrSnFTRVVyYmtsb00wSklTRTV5TkZVd01tNTVjMGhyVUdWcFVRbzVMMDFPWmsxVVNHTndSRmRKYWpsUVZuRlhWbVIzUFQwS0xTMHRMUzFGVGtRZ1EwVlNWRWxHU1VOQlZFVXRMUzB0TFFvPSJ9fX19"}], "timestampVerificationData":{"rfc3161Timestamps":[{"signedTimestamp":"MIICyjADAgEAMIICwQYJKoZIhvcNAQcCoIICsjCCAq4CAQMxDTALBglghkgBZQMEAgEwgbgGCyqGSIb3DQEJEAEEoIGoBIGlMIGiAgEBBgkrBgEEAYO/MAIwMTANBglghkgBZQMEAgEFAAQgMV27ny/pRdg0VJKN5fqO9lOFJmiWNT3rxaIPz1wkaSkCFQCc3tEnstHOa0uhAALXWj6oNlC8qxgPMjAyNjEwMTkyMjMxMTlaMAMCAQGgMqQwMC4xFTATBgNVBAoTDHNpZ3N0b3JlLmRldjEVMBMGA1UEAxMMc2lnc3RvcmUtdHNhoAAxggHbMIIB1wIBATBRMDkxFTATBgNVBAoTDHNpZ3N0b3JlLmRldjEgMB4GA1UEAxMXc2lnc3RvcmUtdHNhLXNlbGZzaWduZWQCFDoTVC8MkGHuvMFDL8uKjosqI4sMMAsGCWCGSAFlAwQCAaCB/DAaBgkqhkiG9w0BCQMxDQYLKoZIhvcNAQkQAQQwHAYJKoZIhvcNAQkFMQ8XDTI2MTAxOTIyMzExOVowLwYJKoZIhvcNAQkEMSIEIJHVyEK3fysoP8hsyCWsOrMfWZMQOfqWTlN9KN21lE4gMIGOBgsqhkiG9w0BCRACLzF/MH0wezB5BCCF+Se8B6tiysO0Q1bBDvyBssaIP9p6uebYcNnROs0FtzBVMD2kOzA5MRUwEwYDVQQKEwxzaWdzdG9yZS5kZXYxIDAeBgNVBAMTF3NpZ3N0b3JlLXRzYS1zZWxmc2lnbmVkAhQ6E1QvDJBh7rzBQy/Lio6LKiOLDDAKBggqhkjOPQQDAgRnMGUCMDDXZ0VmXJa1mJXl/Wx/vvMDWqueWMyUYK4rPVkU5rIBZTNsWRCtJk9OuhfccKYMdQIxALzWf9ROpy2FjItS02afzRuAcHdLESnQzThjpLb1PU0z6HR6pTPo6XHEr+nvmP2muA=="}]}}, "messageSignature":{"messageDigest":{"algorithm":"SHA2_256", "digest":"4KnrSGjKsx0zeEC92xCkHnHpzpoLay6lv9lfbIusNrc="}, "signature":"MEYCIQDwzCyZ/kCJI23CWDRRakDw0THz2iWfNS8ZF/TcU8XXqgIhAI+t1cg4hU61qchRo1UiuJ/kXmw6m5MKK8zH5YUriRO3"}} \ No newline at end of file diff --git a/build/torch213-rocm72-x86_64-linux/mha.py b/build/torch213-rocm72-x86_64-linux/mha.py new file mode 100644 index 0000000000000000000000000000000000000000..01e2d6a0fbee98d27129f2eb695ac38df73e6487 --- /dev/null +++ b/build/torch213-rocm72-x86_64-linux/mha.py @@ -0,0 +1,332 @@ +"""FlashAttention forward entry points backed by aiter's Composable-Kernel FMHA. + +This is the compiled (HIP / Composable Kernel) counterpart to the Triton +``aiter-flash-attn`` package. Only the *forward* pass is provided (dense and +variable-length), including support for learnable attention sinks via +``window_size[2]`` / ``s_aux`` (e.g. gpt-oss). The op set ships exclusively +the CK ``mha_fwd`` / ``mha_varlen_fwd`` kernels, so the sink path is always +taken (no fall-through to a non-sink ASM kernel). +""" + +from typing import Optional, Tuple, Union + +import torch +from torch import Tensor + +from ._ops import ops + + +def _round_multiple(x: int, m: int) -> int: + return (x + m - 1) // m * m + + +def _maybe_pad_headdim(t: Tensor, target: int) -> Tensor: + og = t.size(-1) + if og == target: + return t + return torch.nn.functional.pad(t, [0, target - og]) + + +def _unpack_window(window_size) -> Tuple[int, int, int]: + left = int(window_size[0]) + right = int(window_size[1]) + sink = int(window_size[2]) if len(window_size) == 3 else 0 + return left, right, sink + + +def _prep_sink(s_aux: Optional[Tensor], nheads: int, device) -> Optional[Tensor]: + if s_aux is None: + return None + assert s_aux.device == device, "s_aux must be on the same device as q" + assert s_aux.shape[0] == nheads, "s_aux must have shape [nheads_q]" + if s_aux.dtype != torch.float32: + s_aux = s_aux.to(torch.float32) + return s_aux + + +def _effective_sink_size(sink_size: int, s_aux: Optional[Tensor]) -> int: + # The CK kernel only takes the sink path when sink_size > 0. Callers using + # the transformers attention interface pass the per-head sink logits as + # ``s_aux`` but leave ``window_size`` a 2-tuple (sink_size == 0), so enable + # a single learnable sink automatically when s_aux is given. + if s_aux is not None and sink_size == 0: + return 1 + return sink_size + + +def flash_attn_func( + q: Tensor, + k: Tensor, + v: Tensor, + dropout_p: float = 0.0, + softmax_scale: Optional[float] = None, + causal: bool = False, + window_size: Tuple[int, ...] = (-1, -1, 0), + bias: Optional[Tensor] = None, + alibi_slopes: Optional[Tensor] = None, + return_lse: bool = False, + return_attn_probs: bool = False, + s_aux: Optional[Tensor] = None, + **kwargs, +) -> Union[Tensor, Tuple[Tensor, ...]]: + """Dense FlashAttention forward. + + ``q, k, v`` have shape ``(batch, seqlen, nheads, headdim)``. ``window_size`` + is ``(left, right, sink_size)``; ``s_aux`` is an optional ``(nheads_q,)`` + fp32 tensor of learnable sink logits. Extra keyword arguments (e.g. those + passed by the transformers flash-attention interface) are ignored. + """ + if softmax_scale is None: + softmax_scale = q.shape[-1] ** (-0.5) + window_size_left, window_size_right, sink_size = _unpack_window(window_size) + + nheads_q = q.size(2) + s_aux = _prep_sink(s_aux, nheads_q, q.device) + sink_size = _effective_sink_size(sink_size, s_aux) + + head_q_og = q.size(-1) + head_v_og = v.size(-1) + head_q = _round_multiple(head_q_og, 8) + head_v = _round_multiple(head_v_og, 8) + q = _maybe_pad_headdim(q, head_q) + k = _maybe_pad_headdim(k, head_q) + v = _maybe_pad_headdim(v, head_v) + + return_softmax = return_attn_probs and dropout_p > 0 + out, softmax_lse, S_dmask, _ = ops.mha_fwd( + q, + k, + v, + dropout_p, + softmax_scale, + causal, + window_size_left, + window_size_right, + sink_size, + return_lse or return_attn_probs, + return_softmax, + None, # cu_seqlens_q + None, # cu_seqlens_kv + None, # out + bias, + alibi_slopes, + None, # q_descale + None, # k_descale + None, # v_descale + s_aux, + None, # gen + ) + + out = out[..., :head_v_og] + if not (return_lse or return_attn_probs): + return out + result = (out,) + if return_lse: + result = result + (softmax_lse,) + if return_attn_probs: + result = result + (S_dmask,) + return result + + +def flash_attn_varlen_func( + q: Tensor, + k: Tensor, + v: Tensor, + cu_seqlens_q: Tensor, + cu_seqlens_k: Tensor, + max_seqlen_q: int, + max_seqlen_k: int, + dropout_p: float = 0.0, + softmax_scale: Optional[float] = None, + causal: bool = False, + window_size: Tuple[int, ...] = (-1, -1, 0), + bias: Optional[Tensor] = None, + alibi_slopes: Optional[Tensor] = None, + return_lse: bool = False, + return_attn_probs: bool = False, + block_table: Optional[Tensor] = None, + min_seqlen_q: int = 0, + logits_soft_cap: float = 0.0, + zero_tensors: bool = False, + cu_seqlens_q_padded: Optional[Tensor] = None, + cu_seqlens_k_padded: Optional[Tensor] = None, + s_aux: Optional[Tensor] = None, + **kwargs, +) -> Union[Tensor, Tuple[Tensor, ...]]: + """Variable-length / packed FlashAttention forward. + + ``q`` has shape ``(total_q, nheads, headdim)``; ``cu_seqlens_*`` are int32 + cumulative offsets of shape ``(batch + 1,)``. Sinks are supported through + ``window_size[2]`` / ``s_aux`` as in :func:`flash_attn_func`. Extra keyword + arguments (e.g. from the transformers flash-attention interface) are ignored. + """ + if softmax_scale is None: + softmax_scale = q.shape[-1] ** (-0.5) + window_size_left, window_size_right, sink_size = _unpack_window(window_size) + + nheads_q = q.size(1) + s_aux = _prep_sink(s_aux, nheads_q, q.device) + sink_size = _effective_sink_size(sink_size, s_aux) + + head_q_og = q.size(-1) + head_v_og = v.size(-1) + head_q = _round_multiple(head_q_og, 8) + head_v = _round_multiple(head_v_og, 8) + q = _maybe_pad_headdim(q, head_q) + k = _maybe_pad_headdim(k, head_q) + v = _maybe_pad_headdim(v, head_v) + + return_softmax = return_attn_probs and dropout_p > 0 + out, softmax_lse, S_dmask, _ = ops.mha_varlen_fwd( + q, + k, + v, + cu_seqlens_q, + cu_seqlens_k, + max_seqlen_q, + max_seqlen_k, + min_seqlen_q, + dropout_p, + softmax_scale, + logits_soft_cap, + zero_tensors, + causal, + window_size_left, + window_size_right, + sink_size, + return_lse or return_attn_probs, + return_softmax, + None, # out + block_table, + bias, + alibi_slopes, + None, # q_descale + None, # k_descale + None, # v_descale + None, # gen + cu_seqlens_q_padded, + cu_seqlens_k_padded, + s_aux, + ) + + out = out[..., :head_v_og] + if not (return_lse or return_attn_probs): + return out + result = (out,) + if return_lse: + result = result + (softmax_lse,) + if return_attn_probs: + result = result + (S_dmask,) + return result + + +def _append_to_contiguous_cache(k_cache, v_cache, k, v, cache_seqlens, Sq): + """In-place append of new k/v into a contiguous cache at per-batch offsets.""" + B = k_cache.shape[0] + if isinstance(cache_seqlens, int): + cache_seqlens = torch.full((B,), cache_seqlens, device=k_cache.device, dtype=torch.long) + pos = cache_seqlens.view(B, 1).to(torch.long) + torch.arange(Sq, device=k_cache.device).view(1, Sq) + bidx = torch.arange(B, device=k_cache.device).view(B, 1).expand(B, Sq) + k_cache[bidx, pos] = k.to(k_cache.dtype) + v_cache[bidx, pos] = v.to(v_cache.dtype) + + +def flash_attn_with_kvcache( + q: Tensor, + k_cache: Tensor, + v_cache: Tensor, + k: Optional[Tensor] = None, + v: Optional[Tensor] = None, + rotary_cos: Optional[Tensor] = None, + rotary_sin: Optional[Tensor] = None, + cache_seqlens=None, + block_table: Optional[Tensor] = None, + softmax_scale: Optional[float] = None, + causal: bool = True, + window_size: Tuple[int, ...] = (-1, -1, 0), + s_aux: Optional[Tensor] = None, + return_lse: bool = False, + **kwargs, +) -> Union[Tensor, Tuple[Tensor, ...]]: + """FlashAttention against a KV cache, built on the CK split-KV kernel. + + ``q`` is ``(batch, seqlen_q, nheads, headdim)``. For a contiguous cache, + ``k_cache``/``v_cache`` are ``(batch, seqlen_cache, nheads_k, headdim)`` and + ``cache_seqlens`` (int or ``(batch,)``) is the number of valid cache entries + *before* appending. If ``k``/``v`` (the new tokens) are given, they are + written into the cache in place. The query attends to the valid prefix + (``cache_seqlens`` + ``seqlen_q``) with bottom-right causal alignment. + + Notes / current limitations: + - Rotary embedding is not applied here; pass ``q``/``k`` already rotated + (``rotary_cos``/``rotary_sin`` are rejected). + - Paged caches are addressed via ``block_table`` (forwarded to the split-KV + varlen kernel); in-cache append for paged layouts must be done by the + caller (pass ``k=v=None``). + """ + if rotary_cos is not None or rotary_sin is not None: + raise NotImplementedError( + "flash_attn_with_kvcache does not apply rotary; rotate q/k beforehand." + ) + if softmax_scale is None: + softmax_scale = q.shape[-1] ** (-0.5) + window_size_left, window_size_right, sink_size = _unpack_window(window_size) + B, Sq, Hq, D = q.shape + s_aux = _prep_sink(s_aux, Hq, q.device) + sink_size = _effective_sink_size(sink_size, s_aux) + + paged = block_table is not None + + # ---- append new tokens into the cache (contiguous layout only) ---- + if k is not None and not paged: + assert cache_seqlens is not None, "cache_seqlens required to append into the cache" + _append_to_contiguous_cache(k_cache, v_cache, k, v, cache_seqlens, Sq) + elif k is not None and paged: + raise NotImplementedError( + "in-cache append for paged layouts is not supported; append externally " + "and call with k=v=None." + ) + + appended = 0 if k is None else Sq + + # ---- key lengths ---- + if cache_seqlens is None: + klens = torch.full((B,), k_cache.shape[1], device=q.device, dtype=torch.long) + elif isinstance(cache_seqlens, int): + klens = torch.full((B,), cache_seqlens + appended, device=q.device, dtype=torch.long) + else: + klens = cache_seqlens.to(q.device, torch.long) + appended + + uniform = bool((klens == klens[0]).all().item()) + + # ---- fast path: contiguous cache, uniform length -> dense kernel ---- + if not paged and uniform: + L = int(klens[0].item()) + return flash_attn_func( + q, k_cache[:, :L], v_cache[:, :L], + softmax_scale=softmax_scale, causal=causal, + window_size=(window_size_left, window_size_right, sink_size), + s_aux=s_aux, return_lse=return_lse, + ) + + # ---- general path: variable cache lengths -> split-KV varlen kernel ---- + cu_q = torch.arange(0, (B + 1) * Sq, Sq, device=q.device, dtype=torch.int32) + cu_k = torch.zeros(B + 1, device=q.device, dtype=torch.int32) + cu_k[1:] = torch.cumsum(klens, 0).to(torch.int32) + q_packed = q.reshape(B * Sq, Hq, D) + if paged: + k_packed, v_packed = k_cache, v_cache # kernel gathers via block_table + else: + k_packed = torch.cat([k_cache[b, : int(klens[b])] for b in range(B)], dim=0) + v_packed = torch.cat([v_cache[b, : int(klens[b])] for b in range(B)], dim=0) + + out = flash_attn_varlen_func( + q_packed, k_packed, v_packed, cu_q, cu_k, Sq, int(klens.max().item()), + softmax_scale=softmax_scale, causal=causal, + window_size=(window_size_left, window_size_right, sink_size), + block_table=block_table, s_aux=s_aux, return_lse=return_lse, + ) + if return_lse: + out, lse = out + return out.reshape(B, Sq, Hq, D), lse + return out.reshape(B, Sq, Hq, D) diff --git a/build/torch213-rocm72-x86_64-linux/symbols.json b/build/torch213-rocm72-x86_64-linux/symbols.json new file mode 100644 index 0000000000000000000000000000000000000000..e8348db4bd94712352e7fb3e83b1d94f47c2814b --- /dev/null +++ b/build/torch213-rocm72-x86_64-linux/symbols.json @@ -0,0 +1,349 @@ +{ + "schema_version": 1, + "module": "_aiter_flash_attn_ck_rocm_fe2bb66", + "functions": [ + { + "name": "flash_attn_func", + "qualname": "flash_attn_func", + "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha", + "kind": "function", + "doc": "Dense FlashAttention forward.\n\n``q, k, v`` have shape ``(batch, seqlen, nheads, headdim)``. ``window_size``\nis ``(left, right, sink_size)``; ``s_aux`` is an optional ``(nheads_q,)``\nfp32 tensor of learnable sink logits. Extra keyword arguments (e.g. those\npassed by the transformers flash-attention interface) are ignored.", + "signature": { + "parameters": [ + { + "name": "q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "v", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "dropout_p", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "0.0", + "annotation": "float" + }, + { + "name": "softmax_scale", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "float | None" + }, + { + "name": "causal", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "window_size", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "(-1, -1, 0)", + "annotation": "Tuple[int, ...]" + }, + { + "name": "bias", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "alibi_slopes", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "return_lse", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "return_attn_probs", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "s_aux", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "kwargs", + "kind": "VAR_KEYWORD", + "default": null, + "annotation": null + } + ], + "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]" + } + }, + { + "name": "flash_attn_varlen_func", + "qualname": "flash_attn_varlen_func", + "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha", + "kind": "function", + "doc": "Variable-length / packed FlashAttention forward.\n\n``q`` has shape ``(total_q, nheads, headdim)``; ``cu_seqlens_*`` are int32\ncumulative offsets of shape ``(batch + 1,)``. Sinks are supported through\n``window_size[2]`` / ``s_aux`` as in :func:`flash_attn_func`. Extra keyword\narguments (e.g. from the transformers flash-attention interface) are ignored.", + "signature": { + "parameters": [ + { + "name": "q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "v", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "cu_seqlens_q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "cu_seqlens_k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "max_seqlen_q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "int" + }, + { + "name": "max_seqlen_k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "int" + }, + { + "name": "dropout_p", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "0.0", + "annotation": "float" + }, + { + "name": "softmax_scale", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "float | None" + }, + { + "name": "causal", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "window_size", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "(-1, -1, 0)", + "annotation": "Tuple[int, ...]" + }, + { + "name": "bias", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "alibi_slopes", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "return_lse", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "return_attn_probs", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "block_table", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "min_seqlen_q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "0", + "annotation": "int" + }, + { + "name": "logits_soft_cap", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "0.0", + "annotation": "float" + }, + { + "name": "zero_tensors", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "cu_seqlens_q_padded", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "cu_seqlens_k_padded", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "s_aux", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "kwargs", + "kind": "VAR_KEYWORD", + "default": null, + "annotation": null + } + ], + "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]" + } + }, + { + "name": "flash_attn_with_kvcache", + "qualname": "flash_attn_with_kvcache", + "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha", + "kind": "function", + "doc": "FlashAttention against a KV cache, built on the CK split-KV kernel.\n\n``q`` is ``(batch, seqlen_q, nheads, headdim)``. For a contiguous cache,\n``k_cache``/``v_cache`` are ``(batch, seqlen_cache, nheads_k, headdim)`` and\n``cache_seqlens`` (int or ``(batch,)``) is the number of valid cache entries\n*before* appending. If ``k``/``v`` (the new tokens) are given, they are\nwritten into the cache in place. The query attends to the valid prefix\n(``cache_seqlens`` + ``seqlen_q``) with bottom-right causal alignment.\n\nNotes / current limitations:\n- Rotary embedding is not applied here; pass ``q``/``k`` already rotated\n (``rotary_cos``/``rotary_sin`` are rejected).\n- Paged caches are addressed via ``block_table`` (forwarded to the split-KV\n varlen kernel); in-cache append for paged layouts must be done by the\n caller (pass ``k=v=None``).", + "signature": { + "parameters": [ + { + "name": "q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "k_cache", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "v_cache", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "v", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "rotary_cos", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "rotary_sin", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "cache_seqlens", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": null + }, + { + "name": "block_table", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "softmax_scale", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "float | None" + }, + { + "name": "causal", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "True", + "annotation": "bool" + }, + { + "name": "window_size", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "(-1, -1, 0)", + "annotation": "Tuple[int, ...]" + }, + { + "name": "s_aux", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "return_lse", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "kwargs", + "kind": "VAR_KEYWORD", + "default": null, + "annotation": null + } + ], + "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]" + } + } + ], + "layers": [] +} diff --git a/build/torch214-rocm714-x86_64-linux/__init__.py b/build/torch214-rocm714-x86_64-linux/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..73acb58c1142a1ab2a6ed9d6f34bc6a53ab5e9b5 --- /dev/null +++ b/build/torch214-rocm714-x86_64-linux/__init__.py @@ -0,0 +1,27 @@ +"""AITER Composable-Kernel FlashAttention kernels for AMD ROCm (compiled HIP). + +Compiled (Composable Kernel / HIP) counterpart to the Triton-based +``aiter-flash-attn`` package, repackaged from the CK FMHA forward path of the +`ROCm/aiter `_ project as a self-contained +Hugging Face Hub kernel. Exposes the FlashAttention forward entry points with +learnable attention sink support (e.g. gpt-oss). +""" + +from .mha import ( + flash_attn_func, + flash_attn_varlen_func, + flash_attn_with_kvcache, +) + + +__kernel_metadata__ = { + "license": "mit", +} + + +__all__ = [ + "__kernel_metadata__", + "flash_attn_func", + "flash_attn_varlen_func", + "flash_attn_with_kvcache", +] diff --git a/build/torch214-rocm714-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so b/build/torch214-rocm714-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so new file mode 100644 index 0000000000000000000000000000000000000000..611cce9ffdbc435e4321f43a74c7ef3b857ab7f1 --- /dev/null +++ b/build/torch214-rocm714-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b039b591061456fc9e0001c529f8eacd5432573669ae31721dc9f3aa1605fd8 +size 159738472 diff --git a/build/torch214-rocm714-x86_64-linux/_ops.py b/build/torch214-rocm714-x86_64-linux/_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..2b4dd56c220b61fd2c16e0c1a02ee91a491cef0c --- /dev/null +++ b/build/torch214-rocm714-x86_64-linux/_ops.py @@ -0,0 +1,9 @@ +import torch +from . import _aiter_flash_attn_ck_rocm_fe2bb66 +ops = torch.ops._aiter_flash_attn_ck_rocm_fe2bb66 + +def add_op_namespace_prefix(op_name: str): + """ + Prefix op by namespace. + """ + return f"_aiter_flash_attn_ck_rocm_fe2bb66::{op_name}" diff --git a/build/torch214-rocm714-x86_64-linux/metadata.json b/build/torch214-rocm714-x86_64-linux/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..0823599a4e1892c7f8c48253d84f2103cc24d4ff --- /dev/null +++ b/build/torch214-rocm714-x86_64-linux/metadata.json @@ -0,0 +1,37 @@ +{ + "name": "aiter-flash-attn-ck", + "id": "_aiter_flash_attn_ck_rocm_fe2bb66", + "version": 1, + "kernels-minver": "0.14.0", + "license": "MIT", + "python-depends": [], + "kernel-depends": [], + "backend": { + "type": "rocm", + "archs": [ + "gfx942", + "gfx950" + ] + }, + "digest": { + "algorithm": "sha256", + "files": { + "__init__.py": "7wj1GTm2bqTLLCyT4xLuAk/+JWYrhINjY/ODMNWhgZ0=", + "_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so": "SwObWRBhRW/J4AAcUp+OrNVDJXNmmuMXIdyfOqFgX9g=", + "_ops.py": "4WkdYDWWJp60zep+oW/5ym9vIeVMNfrgRCkP3o/w7yU=", + "mha.py": "zCAjY4O+DiCzmEL+W9sqXxZaqjsj4RJbWFcyPpUZejk=", + "symbols.json": "rhnF4H8Jsb6I+YPxIVFMgnLcTQJODEXbZ7ge77NZXAU=" + } + }, + "provenance": { + "kernel-builder": { + "version": "0.18.0-dev0", + "commit": "d054906a13710b811b96697b4635f03020842733", + "dirty": false + }, + "kernel": { + "commit": "fe2bb66fee6c2a77eccf08a0508dc57428d488ed", + "dirty": false + } + } +} \ No newline at end of file diff --git a/build/torch214-rocm714-x86_64-linux/metadata.json.sigstore b/build/torch214-rocm714-x86_64-linux/metadata.json.sigstore new file mode 100644 index 0000000000000000000000000000000000000000..28ebbf07905184ebdd3cdb3c2815db80623a7752 --- /dev/null +++ b/build/torch214-rocm714-x86_64-linux/metadata.json.sigstore @@ -0,0 +1 @@ +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json", "verificationMaterial":{"certificate":{"rawBytes":"MIIHLTCCBrOgAwIBAgIUAiMLJSAt90xH9435AyuCDCgEzv0wCgYIKoZIzj0EAwMwNzEVMBMGA1UEChMMc2lnc3RvcmUuZGV2MR4wHAYDVQQDExVzaWdzdG9yZS1pbnRlcm1lZGlhdGUwHhcNMjYxMDE5MjIzMTIwWhcNMjYxMDE5MjI0MTIwWjAAMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEVHCNYqrFGpiLcikuWEN3eqflc+RdNyTdEAEtm0aHaRGxG2O/y9XYYhklxUCkZfhSC6CKIkWeg128/wqOfrRcG6OCBdIwggXOMA4GA1UdDwEB/wQEAwIHgDATBgNVHSUEDDAKBggrBgEFBQcDAzAfBgNVHSMEGDAWgBTf0+nPViQRlvmo2OkoVaLGLhhkPzBrBgNVHREBAf8EYTBfhl1odHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkvLmdpdGh1Yi93b3JrZmxvd3MvYnVpbGQueWFtbEByZWZzL2hlYWRzL21haW4wOQYKKwYBBAGDvzABAQQraHR0cHM6Ly90b2tlbi5hY3Rpb25zLmdpdGh1YnVzZXJjb250ZW50LmNvbTAfBgorBgEEAYO/MAECBBF3b3JrZmxvd19kaXNwYXRjaDA2BgorBgEEAYO/MAEDBChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMBMGCisGAQQBg78wAQQEBUJ1aWxkMCsGCisGAQQBg78wAQUEHWh1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5MB0GCisGAQQBg78wAQYED3JlZnMvaGVhZHMvbWFpbjA7BgorBgEEAYO/MAEIBC0MK2h0dHBzOi8vdG9rZW4uYWN0aW9ucy5naXRodWJ1c2VyY29udGVudC5jb20wbQYKKwYBBAGDvzABCQRfDF1odHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkvLmdpdGh1Yi93b3JrZmxvd3MvYnVpbGQueWFtbEByZWZzL2hlYWRzL21haW4wOAYKKwYBBAGDvzABCgQqDChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMBsGCisGAQQBg78wAQsEDQwLc2VsZi1ob3N0ZWQwQAYKKwYBBAGDvzABDAQyDDBodHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkwOAYKKwYBBAGDvzABDQQqDChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMB8GCisGAQQBg78wAQ4EEQwPcmVmcy9oZWFkcy9tYWluMBoGCisGAQQBg78wAQ8EDAwKMTA3MTQ3NTUyOTAuBgorBgEEAYO/MAEQBCAMHmh0dHBzOi8vZ2l0aHViLmNvbS9odWdnaW5nZmFjZTAYBgorBgEEAYO/MAERBAoMCDI1NzIwNzQzMG0GCisGAQQBg78wARIEXwxdaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5Ly5naXRodWIvd29ya2Zsb3dzL2J1aWxkLnlhbWxAcmVmcy9oZWFkcy9tYWluMDgGCisGAQQBg78wARMEKgwoZmUyYmI2NmZlZTZjMmE3N2VjY2YwOGEwNTA4ZGM1NzQyOGQ0ODhlZDAhBgorBgEEAYO/MAEUBBMMEXdvcmtmbG93X2Rpc3BhdGNoMGQGCisGAQQBg78wARUEVgxUaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5L2FjdGlvbnMvcnVucy8zOTIzOTUxODEwNC9hdHRlbXB0cy8xMBYGCisGAQQBg78wARYECAwGcHVibGljMEYGCisGAQQBg78wARgEOAw2cmVwbzpodWdnaW5nZmFjZS9rZXJuZWxzLWNvbW11bml0eTpyZWY6cmVmcy9oZWFkcy9tYWluMIGLBgorBgEEAdZ5AgQCBH0EewB5AHcA3T0wasbHETJjGR4cmWc3AqJKXrjePK3/h4pygC8p7o4AAAGhVkpskwAABAMASDBGAiEAjR6Edox26woSK8VUoTrbhCZSZan25Orrok0qMsZN848CIQCGnFTHPiudFCLy87Kb7nkzec9kYBp6xCiFdA2haKd0KTAKBggqhkjOPQQDAwNoADBlAjAvKAQZRf1tYZQYszQcDHgS9KFDfyv01th/6lYg1/Lk2xxVB13UZhpdpqmPBuOLILYCMQC49Z9qmGO+nRlV6rKlHVCcxfOCKuIMBg7deqJMPBoHU4imDGvOEwq6LoTsJ4Yg3W0="}, "tlogEntries":[{"logIndex":"3327212529", "logId":{"keyId":"wNI9atQGlz+VWfO6LRygH4QUfY/8W4RFwiT5i5WRgB0="}, "kindVersion":{"kind":"hashedrekord", "version":"0.0.1"}, "integratedTime":"1792449080", "inclusionPromise":{"signedEntryTimestamp":"MEUCIQD8GpHM6MKXR/rptxayIf3aE1VpnCLfuQhhHttxI2aaRgIgEqBQDefddfyYmc5RdCw8yznGdOSO0FMiZ5/YXbfUVKQ="}, "inclusionProof":{"logIndex":"3205308267", "rootHash":"ZwNWKafWDGIa7jS322+9hAERxjV8VqfzNToR/bqhbO8=", "treeSize":"3205308308", "hashes":["GkSR5JmwSROMgQp+X+Bn6XNlu/QCI6BEeqOd7xHIx/E=", "SNg1FvcD101ISb8Dg2WTHg8pjd4rL3Yin2HOLXA+JNU=", "kqbleeqHhUh+wWimGg0L6YW/mEPSaTmH57ZkIWP9pDs=", "sUdc7htS9xVri0j+QhVufsXLQ//Cl8EOSMNRSHJN2V8=", "EWnA50KfGQ7OSrzfWyRMpfD8fzqvTBqodiVike3PY+U=", "OuFj11vY/wkFgf4GOMxqGrz4vbz8i/wpTynK/TFaeuM=", "r+NHFu6pG7y+N8cdHz5BBKWan8TceJvMQefQC6248zI=", "BB/vu1m25hn/VARTtokC4JRms4iMok+HhUzZi9zIbDg=", "jM0mX0eoGvIzHLbWcS4NqOxDrG+c3tvgXFr5h3Xnz58=", "gBGf/lDkk/5tDR8CLTuE5Pt2YTL1zjLs12T3vvxq7X4=", "0aSE/O6jCq82vSwoybvpblRnnP72+1gEInKy6Wyc3E0=", "5egUlZ4Pi0Ix/qLyCDXocsWywJCHqrde/S06vOZvlOM=", "QsXufUpoKtSjKAB19K1G00umQpWvPHlBNuRGFiiVjQQ=", "QIMkwQbwAewhLJd//5FFwI3SnyNQBGjCgJrelfG8vhc=", "Et17CACUv9QN64irUJuBjsrgRGkGnnJ9xacRV/aBdwQ=", "lqgEwkxhDMs/gdIAE4lmjfMwg7nDmv7lda6O0LGliCw=", "HovTPBWzoErvUg1u5sIi3x1ZonZ7yJekVsBrmpomCIA=", "G+nk7mgE5h6dUmv2UY7NDJrY3WvrZY31raaO1AETt+o=", "qlTWNDzERnTBQVcHHXXVPjiUP0SsikGxfcxVlgqVFDo=", "gg9GNApNke3+zNhl4tsWtmIycCFrOMCB3lys6XuSQNQ=", "NlOclldmKPA1tMKIskqHAwuQDYHp31YwpK+OOMrMccc=", "qxzHanAzz57SDdmJe0B7bJK72NTIBbwEMGBBKvDOROw=", "xH/DCseLHr9eKoYT8qsORZK7zVdEGYWHuVtsVrD95wY="], "checkpoint":{"envelope":"rekor.sigstore.dev - 1193050959916656506\n3205308308\nZwNWKafWDGIa7jS322+9hAERxjV8VqfzNToR/bqhbO8=\n\n— rekor.sigstore.dev wNI9ajBEAiBd+kYHNkuSNKFM4FXs+0qVPSTkEfqy5yT7knlRqbvh3AIgC0Sm92N4g09V83aAMO5QUnQHru8C+lKvq6wUAnHowsg=\n"}}, "canonicalizedBody":"eyJhcGlWZXJzaW9uIjoiMC4wLjEiLCJraW5kIjoiaGFzaGVkcmVrb3JkIiwic3BlYyI6eyJkYXRhIjp7Imhhc2giOnsiYWxnb3JpdGhtIjoic2hhMjU2IiwidmFsdWUiOiJlYjY3ZjhjZTEyNDBkMDUwYTYzNTdjOWJkNWMwYzZiYjA3Mzk3ZTcwYzhkMWE0MDg1ODc0ZTc5MjQ3YjlmZjVmIn19LCJzaWduYXR1cmUiOnsiY29udGVudCI6Ik1FVUNJRnJMRlhMZjBQaXZKR2xYS0JHKzMxVHd0d3dJR1ZYK2NUSTRVMDc5TUZpbkFpRUEvdDd0djBNZkJkcGIzaVBFSmxadlpBeXZ5cisrMXlFbkVvWm9HQUsxbUxNPSIsInB1YmxpY0tleSI6eyJjb250ZW50IjoiTFMwdExTMUNSVWRKVGlCRFJWSlVTVVpKUTBGVVJTMHRMUzB0Q2sxSlNVaE1WRU5EUW5KUFowRjNTVUpCWjBsVlFXbE5URXBUUVhRNU1IaElPVFF6TlVGNWRVTkVRMmRGZW5Zd2QwTm5XVWxMYjFwSmVtb3dSVUYzVFhjS1RucEZWazFDVFVkQk1WVkZRMmhOVFdNeWJHNWpNMUoyWTIxVmRWcEhWakpOVWpSM1NFRlpSRlpSVVVSRmVGWjZZVmRrZW1SSE9YbGFVekZ3WW01U2JBcGpiVEZzV2tkc2FHUkhWWGRJYUdOT1RXcFplRTFFUlRWTmFrbDZUVlJKZDFkb1kwNU5hbGw0VFVSRk5VMXFTVEJOVkVsM1YycEJRVTFHYTNkRmQxbElDa3R2V2tsNmFqQkRRVkZaU1V0dldrbDZhakJFUVZGalJGRm5RVVZXU0VOT1dYRnlSa2R3YVV4amFXdDFWMFZPTTJWeFpteGpLMUprVG5sVVpFVkJSWFFLYlRCaFNHRlNSM2hITWs4dmVUbFlXVmxvYTJ4NFZVTnJXbVpvVTBNMlEwdEphMWRsWnpFeU9DOTNjVTltY2xKalJ6WlBRMEprU1hkbloxaFBUVUUwUndwQk1WVmtSSGRGUWk5M1VVVkJkMGxJWjBSQlZFSm5UbFpJVTFWRlJFUkJTMEpuWjNKQ1owVkdRbEZqUkVGNlFXWkNaMDVXU0ZOTlJVZEVRVmRuUWxSbUNqQXJibEJXYVZGU2JIWnRiekpQYTI5V1lVeEhUR2hvYTFCNlFuSkNaMDVXU0ZKRlFrRm1PRVZaVkVKbWFHd3hiMlJJVW5kamVtOTJUREprY0dSSGFERUtXV2sxYW1JeU1IWmhTRlp1V2pKc2RWb3lXbWhaTWxWMllUSldlV0p0Vm5OamVURnFZakl4ZEdSWE5YQmtTR3QyVEcxa2NHUkhhREZaYVRrellqTktjZ3BhYlhoMlpETk5kbGx1Vm5CaVIxRjFaVmRHZEdKRlFubGFWMXA2VERKb2JGbFhVbnBNTWpGb1lWYzBkMDlSV1V0TGQxbENRa0ZIUkhaNlFVSkJVVkZ5Q21GSVVqQmpTRTAyVEhrNU1HSXlkR3hpYVRWb1dUTlNjR0l5TlhwTWJXUndaRWRvTVZsdVZucGFXRXBxWWpJMU1GcFhOVEJNYlU1MllsUkJaa0puYjNJS1FtZEZSVUZaVHk5TlFVVkRRa0pHTTJJelNuSmFiWGgyWkRFNWEyRllUbmRaV0ZKcVlVUkJNa0puYjNKQ1owVkZRVmxQTDAxQlJVUkNRMmh0V2xSS2FRcFphbGt5V20xV2JFNXRUWGxaVkdNeldsZE9hbHBxUVRSWlZFRXhUVVJvYTFsNlZUTk9SRWswV2tSUk5FOUhWbXROUWsxSFEybHpSMEZSVVVKbk56aDNDa0ZSVVVWQ1ZVb3hZVmQ0YTAxRGMwZERhWE5IUVZGUlFtYzNPSGRCVVZWRlNGZG9NVm95WkhCaWJXUnRXVmRPYkV3eWRHeGpiVFZzWWtoTmRGa3lPWFFLWWxoV2RXRllValZOUWpCSFEybHpSMEZSVVVKbk56aDNRVkZaUlVRelNteGFiazEyWVVkV2FGcElUWFppVjBad1ltcEJOMEpuYjNKQ1owVkZRVmxQTHdwTlFVVkpRa013VFVzeWFEQmtTRUo2VDJrNGRtUkhPWEphVnpSMVdWZE9NR0ZYT1hWamVUVnVZVmhTYjJSWFNqRmpNbFo1V1RJNWRXUkhWblZrUXpWcUNtSXlNSGRpVVZsTFMzZFpRa0pCUjBSMmVrRkNRMUZTWmtSR01XOWtTRkozWTNwdmRrd3laSEJrUjJneFdXazFhbUl5TUhaaFNGWnVXakpzZFZveVdtZ0tXVEpWZG1FeVZubGliVlp6WTNreGFtSXlNWFJrVnpWd1pFaHJka3h0WkhCa1IyZ3hXV2s1TTJJelNuSmFiWGgyWkROTmRsbHVWbkJpUjFGMVpWZEdkQXBpUlVKNVdsZGFla3d5YUd4WlYxSjZUREl4YUdGWE5IZFBRVmxMUzNkWlFrSkJSMFIyZWtGQ1EyZFJjVVJEYUcxYVZFcHBXV3BaTWxwdFZteE9iVTE1Q2xsVVl6TmFWMDVxV21wQk5GbFVRVEZOUkdocldYcFZNMDVFU1RSYVJGRTBUMGRXYTAxQ2MwZERhWE5IUVZGUlFtYzNPSGRCVVhORlJGRjNUR015Vm5NS1dta3hiMkl6VGpCYVYxRjNVVUZaUzB0M1dVSkNRVWRFZG5wQlFrUkJVWGxFUkVKdlpFaFNkMk42YjNaTU1tUndaRWRvTVZscE5XcGlNakIyWVVoV2JncGFNbXgxV2pKYWFGa3lWWFpoTWxaNVltMVdjMk41TVdwaU1qRjBaRmMxY0dSSWEzZFBRVmxMUzNkWlFrSkJSMFIyZWtGQ1JGRlJjVVJEYUcxYVZFcHBDbGxxV1RKYWJWWnNUbTFOZVZsVVl6TmFWMDVxV21wQk5GbFVRVEZOUkdocldYcFZNMDVFU1RSYVJGRTBUMGRXYTAxQ09FZERhWE5IUVZGUlFtYzNPSGNLUVZFMFJVVlJkMUJqYlZadFkzazViMXBYUm10amVUbDBXVmRzZFUxQ2IwZERhWE5IUVZGUlFtYzNPSGRCVVRoRlJFRjNTMDFVUVROTlZGRXpUbFJWZVFwUFZFRjFRbWR2Y2tKblJVVkJXVTh2VFVGRlVVSkRRVTFJYldnd1pFaENlazlwT0haYU1td3dZVWhXYVV4dFRuWmlVemx2WkZka2JtRlhOVzVhYlVacUNscFVRVmxDWjI5eVFtZEZSVUZaVHk5TlFVVlNRa0Z2VFVORVNURk9la2wzVG5wUmVrMUhNRWREYVhOSFFWRlJRbWMzT0hkQlVrbEZXSGQ0WkdGSVVqQUtZMGhOTmt4NU9XNWhXRkp2WkZkSmRWa3lPWFJNTW1neFdqSmtjR0p0WkcxWlYwNXNUREowYkdOdE5XeGlTRTEwV1RJNWRHSllWblZoV0ZJMVRIazFiZ3BoV0ZKdlpGZEpkbVF5T1hsaE1scHpZak5rZWt3eVNqRmhWM2hyVEc1c2FHSlhlRUZqYlZadFkzazViMXBYUm10amVUbDBXVmRzZFUxRVowZERhWE5IQ2tGUlVVSm5OemgzUVZKTlJVdG5kMjlhYlZWNVdXMUpNazV0V214YVZGcHFUVzFGTTA0eVZtcFpNbGwzVDBkRmQwNVVRVFJhUjAweFRucFJlVTlIVVRBS1QwUm9iRnBFUVdoQ1oyOXlRbWRGUlVGWlR5OU5RVVZWUWtKTlRVVllaSFpqYlhSdFlrYzVNMWd5VW5Cak0wSm9aRWRPYjAxSFVVZERhWE5IUVZGUlFncG5OemgzUVZKVlJWWm5lRlZoU0ZJd1kwaE5Oa3g1T1c1aFdGSnZaRmRKZFZreU9YUk1NbWd4V2pKa2NHSnRaRzFaVjA1c1RESjBiR050Tld4aVNFMTBDbGt5T1hSaVdGWjFZVmhTTlV3eVJtcGtSMngyWW01TmRtTnVWblZqZVRoNlQxUkplazlVVlhoUFJFVjNUa001YUdSSVVteGlXRUl3WTNrNGVFMUNXVWNLUTJselIwRlJVVUpuTnpoM1FWSlpSVU5CZDBkalNGWnBZa2RzYWsxRldVZERhWE5IUVZGUlFtYzNPSGRCVW1kRlQwRjNNbU50Vm5kaWVuQnZaRmRrYmdwaFZ6VnVXbTFHYWxwVE9YSmFXRXAxV2xkNGVreFhUblppVnpFeFltMXNNR1ZVY0hsYVYxazJZMjFXYldONU9XOWFWMFpyWTNrNWRGbFhiSFZOU1VkTUNrSm5iM0pDWjBWRlFXUmFOVUZuVVVOQ1NEQkZaWGRDTlVGSVkwRXpWREIzWVhOaVNFVlVTbXBIVWpSamJWZGpNMEZ4U2t0WWNtcGxVRXN6TDJnMGNIa0taME00Y0Rkdk5FRkJRVWRvVm10d2MydDNRVUZDUVUxQlUwUkNSMEZwUlVGcVVqWkZaRzk0TWpaM2IxTkxPRlpWYjFSeVltaERXbE5hWVc0eU5VOXljZ3B2YXpCeFRYTmFUamcwT0VOSlVVTkhia1pVU0ZCcGRXUkdRMHg1T0RkTFlqZHVhM3BsWXpscldVSndObmhEYVVaa1FUSm9ZVXRrTUV0VVFVdENaMmR4Q21ocmFrOVFVVkZFUVhkT2IwRkVRbXhCYWtGMlMwRlJXbEptTVhSWldsRlpjM3BSWTBSSVoxTTVTMFpFWm5sMk1ERjBhQzgyYkZsbk1TOU1heko0ZUZZS1FqRXpWVnBvY0dSd2NXMVFRblZQVEVsTVdVTk5VVU0wT1ZvNWNXMUhUeXR1VW14V05uSkxiRWhXUTJONFprOURTM1ZKVFVKbk4yUmxjVXBOVUVKdlNBcFZOR2x0UkVkMlQwVjNjVFpNYjFSelNqUlpaek5YTUQwS0xTMHRMUzFGVGtRZ1EwVlNWRWxHU1VOQlZFVXRMUzB0TFFvPSJ9fX19"}], "timestampVerificationData":{"rfc3161Timestamps":[{"signedTimestamp":"MIICyzADAgEAMIICwgYJKoZIhvcNAQcCoIICszCCAq8CAQMxDTALBglghkgBZQMEAgEwgbgGCyqGSIb3DQEJEAEEoIGoBIGlMIGiAgEBBgkrBgEEAYO/MAIwMTANBglghkgBZQMEAgEFAAQgsPR37roMuByuGl+6SO0oNPTTeTXNSYEjvuhp0vmo0uICFQC95qreNfJioxHfuVzawzfEl0UUuhgPMjAyNjEwMTkyMjMxMjBaMAMCAQGgMqQwMC4xFTATBgNVBAoTDHNpZ3N0b3JlLmRldjEVMBMGA1UEAxMMc2lnc3RvcmUtdHNhoAAxggHcMIIB2AIBATBRMDkxFTATBgNVBAoTDHNpZ3N0b3JlLmRldjEgMB4GA1UEAxMXc2lnc3RvcmUtdHNhLXNlbGZzaWduZWQCFDoTVC8MkGHuvMFDL8uKjosqI4sMMAsGCWCGSAFlAwQCAaCB/DAaBgkqhkiG9w0BCQMxDQYLKoZIhvcNAQkQAQQwHAYJKoZIhvcNAQkFMQ8XDTI2MTAxOTIyMzEyMFowLwYJKoZIhvcNAQkEMSIEIDO3xYYncGMdBknN9/5EL2nPM/94G0LvarK9Cc1xgR4lMIGOBgsqhkiG9w0BCRACLzF/MH0wezB5BCCF+Se8B6tiysO0Q1bBDvyBssaIP9p6uebYcNnROs0FtzBVMD2kOzA5MRUwEwYDVQQKEwxzaWdzdG9yZS5kZXYxIDAeBgNVBAMTF3NpZ3N0b3JlLXRzYS1zZWxmc2lnbmVkAhQ6E1QvDJBh7rzBQy/Lio6LKiOLDDAKBggqhkjOPQQDAgRoMGYCMQCMhSKsehqfpEFUTYrZdWXsJUv05daJZf2Fv2tnr8/GIvuIM3x2tzfjKIqs8GyrguECMQCr18uFuSmYlX/tr0sRmhG/orvoQCDCkIS+5aRlJLepU5aBTORwSMwb51iPKKEb8Lo="}]}}, "messageSignature":{"messageDigest":{"algorithm":"SHA2_256", "digest":"62f4zhJA0FCmNXyb1cDGuwc5fnDI0aQIWHTnkke5/18="}, "signature":"MEUCIFrLFXLf0PivJGlXKBG+31TwtwwIGVX+cTI4U079MFinAiEA/t7tv0MfBdpb3iPEJlZvZAyvyr++1yEnEoZoGAK1mLM="}} \ No newline at end of file diff --git a/build/torch214-rocm714-x86_64-linux/mha.py b/build/torch214-rocm714-x86_64-linux/mha.py new file mode 100644 index 0000000000000000000000000000000000000000..01e2d6a0fbee98d27129f2eb695ac38df73e6487 --- /dev/null +++ b/build/torch214-rocm714-x86_64-linux/mha.py @@ -0,0 +1,332 @@ +"""FlashAttention forward entry points backed by aiter's Composable-Kernel FMHA. + +This is the compiled (HIP / Composable Kernel) counterpart to the Triton +``aiter-flash-attn`` package. Only the *forward* pass is provided (dense and +variable-length), including support for learnable attention sinks via +``window_size[2]`` / ``s_aux`` (e.g. gpt-oss). The op set ships exclusively +the CK ``mha_fwd`` / ``mha_varlen_fwd`` kernels, so the sink path is always +taken (no fall-through to a non-sink ASM kernel). +""" + +from typing import Optional, Tuple, Union + +import torch +from torch import Tensor + +from ._ops import ops + + +def _round_multiple(x: int, m: int) -> int: + return (x + m - 1) // m * m + + +def _maybe_pad_headdim(t: Tensor, target: int) -> Tensor: + og = t.size(-1) + if og == target: + return t + return torch.nn.functional.pad(t, [0, target - og]) + + +def _unpack_window(window_size) -> Tuple[int, int, int]: + left = int(window_size[0]) + right = int(window_size[1]) + sink = int(window_size[2]) if len(window_size) == 3 else 0 + return left, right, sink + + +def _prep_sink(s_aux: Optional[Tensor], nheads: int, device) -> Optional[Tensor]: + if s_aux is None: + return None + assert s_aux.device == device, "s_aux must be on the same device as q" + assert s_aux.shape[0] == nheads, "s_aux must have shape [nheads_q]" + if s_aux.dtype != torch.float32: + s_aux = s_aux.to(torch.float32) + return s_aux + + +def _effective_sink_size(sink_size: int, s_aux: Optional[Tensor]) -> int: + # The CK kernel only takes the sink path when sink_size > 0. Callers using + # the transformers attention interface pass the per-head sink logits as + # ``s_aux`` but leave ``window_size`` a 2-tuple (sink_size == 0), so enable + # a single learnable sink automatically when s_aux is given. + if s_aux is not None and sink_size == 0: + return 1 + return sink_size + + +def flash_attn_func( + q: Tensor, + k: Tensor, + v: Tensor, + dropout_p: float = 0.0, + softmax_scale: Optional[float] = None, + causal: bool = False, + window_size: Tuple[int, ...] = (-1, -1, 0), + bias: Optional[Tensor] = None, + alibi_slopes: Optional[Tensor] = None, + return_lse: bool = False, + return_attn_probs: bool = False, + s_aux: Optional[Tensor] = None, + **kwargs, +) -> Union[Tensor, Tuple[Tensor, ...]]: + """Dense FlashAttention forward. + + ``q, k, v`` have shape ``(batch, seqlen, nheads, headdim)``. ``window_size`` + is ``(left, right, sink_size)``; ``s_aux`` is an optional ``(nheads_q,)`` + fp32 tensor of learnable sink logits. Extra keyword arguments (e.g. those + passed by the transformers flash-attention interface) are ignored. + """ + if softmax_scale is None: + softmax_scale = q.shape[-1] ** (-0.5) + window_size_left, window_size_right, sink_size = _unpack_window(window_size) + + nheads_q = q.size(2) + s_aux = _prep_sink(s_aux, nheads_q, q.device) + sink_size = _effective_sink_size(sink_size, s_aux) + + head_q_og = q.size(-1) + head_v_og = v.size(-1) + head_q = _round_multiple(head_q_og, 8) + head_v = _round_multiple(head_v_og, 8) + q = _maybe_pad_headdim(q, head_q) + k = _maybe_pad_headdim(k, head_q) + v = _maybe_pad_headdim(v, head_v) + + return_softmax = return_attn_probs and dropout_p > 0 + out, softmax_lse, S_dmask, _ = ops.mha_fwd( + q, + k, + v, + dropout_p, + softmax_scale, + causal, + window_size_left, + window_size_right, + sink_size, + return_lse or return_attn_probs, + return_softmax, + None, # cu_seqlens_q + None, # cu_seqlens_kv + None, # out + bias, + alibi_slopes, + None, # q_descale + None, # k_descale + None, # v_descale + s_aux, + None, # gen + ) + + out = out[..., :head_v_og] + if not (return_lse or return_attn_probs): + return out + result = (out,) + if return_lse: + result = result + (softmax_lse,) + if return_attn_probs: + result = result + (S_dmask,) + return result + + +def flash_attn_varlen_func( + q: Tensor, + k: Tensor, + v: Tensor, + cu_seqlens_q: Tensor, + cu_seqlens_k: Tensor, + max_seqlen_q: int, + max_seqlen_k: int, + dropout_p: float = 0.0, + softmax_scale: Optional[float] = None, + causal: bool = False, + window_size: Tuple[int, ...] = (-1, -1, 0), + bias: Optional[Tensor] = None, + alibi_slopes: Optional[Tensor] = None, + return_lse: bool = False, + return_attn_probs: bool = False, + block_table: Optional[Tensor] = None, + min_seqlen_q: int = 0, + logits_soft_cap: float = 0.0, + zero_tensors: bool = False, + cu_seqlens_q_padded: Optional[Tensor] = None, + cu_seqlens_k_padded: Optional[Tensor] = None, + s_aux: Optional[Tensor] = None, + **kwargs, +) -> Union[Tensor, Tuple[Tensor, ...]]: + """Variable-length / packed FlashAttention forward. + + ``q`` has shape ``(total_q, nheads, headdim)``; ``cu_seqlens_*`` are int32 + cumulative offsets of shape ``(batch + 1,)``. Sinks are supported through + ``window_size[2]`` / ``s_aux`` as in :func:`flash_attn_func`. Extra keyword + arguments (e.g. from the transformers flash-attention interface) are ignored. + """ + if softmax_scale is None: + softmax_scale = q.shape[-1] ** (-0.5) + window_size_left, window_size_right, sink_size = _unpack_window(window_size) + + nheads_q = q.size(1) + s_aux = _prep_sink(s_aux, nheads_q, q.device) + sink_size = _effective_sink_size(sink_size, s_aux) + + head_q_og = q.size(-1) + head_v_og = v.size(-1) + head_q = _round_multiple(head_q_og, 8) + head_v = _round_multiple(head_v_og, 8) + q = _maybe_pad_headdim(q, head_q) + k = _maybe_pad_headdim(k, head_q) + v = _maybe_pad_headdim(v, head_v) + + return_softmax = return_attn_probs and dropout_p > 0 + out, softmax_lse, S_dmask, _ = ops.mha_varlen_fwd( + q, + k, + v, + cu_seqlens_q, + cu_seqlens_k, + max_seqlen_q, + max_seqlen_k, + min_seqlen_q, + dropout_p, + softmax_scale, + logits_soft_cap, + zero_tensors, + causal, + window_size_left, + window_size_right, + sink_size, + return_lse or return_attn_probs, + return_softmax, + None, # out + block_table, + bias, + alibi_slopes, + None, # q_descale + None, # k_descale + None, # v_descale + None, # gen + cu_seqlens_q_padded, + cu_seqlens_k_padded, + s_aux, + ) + + out = out[..., :head_v_og] + if not (return_lse or return_attn_probs): + return out + result = (out,) + if return_lse: + result = result + (softmax_lse,) + if return_attn_probs: + result = result + (S_dmask,) + return result + + +def _append_to_contiguous_cache(k_cache, v_cache, k, v, cache_seqlens, Sq): + """In-place append of new k/v into a contiguous cache at per-batch offsets.""" + B = k_cache.shape[0] + if isinstance(cache_seqlens, int): + cache_seqlens = torch.full((B,), cache_seqlens, device=k_cache.device, dtype=torch.long) + pos = cache_seqlens.view(B, 1).to(torch.long) + torch.arange(Sq, device=k_cache.device).view(1, Sq) + bidx = torch.arange(B, device=k_cache.device).view(B, 1).expand(B, Sq) + k_cache[bidx, pos] = k.to(k_cache.dtype) + v_cache[bidx, pos] = v.to(v_cache.dtype) + + +def flash_attn_with_kvcache( + q: Tensor, + k_cache: Tensor, + v_cache: Tensor, + k: Optional[Tensor] = None, + v: Optional[Tensor] = None, + rotary_cos: Optional[Tensor] = None, + rotary_sin: Optional[Tensor] = None, + cache_seqlens=None, + block_table: Optional[Tensor] = None, + softmax_scale: Optional[float] = None, + causal: bool = True, + window_size: Tuple[int, ...] = (-1, -1, 0), + s_aux: Optional[Tensor] = None, + return_lse: bool = False, + **kwargs, +) -> Union[Tensor, Tuple[Tensor, ...]]: + """FlashAttention against a KV cache, built on the CK split-KV kernel. + + ``q`` is ``(batch, seqlen_q, nheads, headdim)``. For a contiguous cache, + ``k_cache``/``v_cache`` are ``(batch, seqlen_cache, nheads_k, headdim)`` and + ``cache_seqlens`` (int or ``(batch,)``) is the number of valid cache entries + *before* appending. If ``k``/``v`` (the new tokens) are given, they are + written into the cache in place. The query attends to the valid prefix + (``cache_seqlens`` + ``seqlen_q``) with bottom-right causal alignment. + + Notes / current limitations: + - Rotary embedding is not applied here; pass ``q``/``k`` already rotated + (``rotary_cos``/``rotary_sin`` are rejected). + - Paged caches are addressed via ``block_table`` (forwarded to the split-KV + varlen kernel); in-cache append for paged layouts must be done by the + caller (pass ``k=v=None``). + """ + if rotary_cos is not None or rotary_sin is not None: + raise NotImplementedError( + "flash_attn_with_kvcache does not apply rotary; rotate q/k beforehand." + ) + if softmax_scale is None: + softmax_scale = q.shape[-1] ** (-0.5) + window_size_left, window_size_right, sink_size = _unpack_window(window_size) + B, Sq, Hq, D = q.shape + s_aux = _prep_sink(s_aux, Hq, q.device) + sink_size = _effective_sink_size(sink_size, s_aux) + + paged = block_table is not None + + # ---- append new tokens into the cache (contiguous layout only) ---- + if k is not None and not paged: + assert cache_seqlens is not None, "cache_seqlens required to append into the cache" + _append_to_contiguous_cache(k_cache, v_cache, k, v, cache_seqlens, Sq) + elif k is not None and paged: + raise NotImplementedError( + "in-cache append for paged layouts is not supported; append externally " + "and call with k=v=None." + ) + + appended = 0 if k is None else Sq + + # ---- key lengths ---- + if cache_seqlens is None: + klens = torch.full((B,), k_cache.shape[1], device=q.device, dtype=torch.long) + elif isinstance(cache_seqlens, int): + klens = torch.full((B,), cache_seqlens + appended, device=q.device, dtype=torch.long) + else: + klens = cache_seqlens.to(q.device, torch.long) + appended + + uniform = bool((klens == klens[0]).all().item()) + + # ---- fast path: contiguous cache, uniform length -> dense kernel ---- + if not paged and uniform: + L = int(klens[0].item()) + return flash_attn_func( + q, k_cache[:, :L], v_cache[:, :L], + softmax_scale=softmax_scale, causal=causal, + window_size=(window_size_left, window_size_right, sink_size), + s_aux=s_aux, return_lse=return_lse, + ) + + # ---- general path: variable cache lengths -> split-KV varlen kernel ---- + cu_q = torch.arange(0, (B + 1) * Sq, Sq, device=q.device, dtype=torch.int32) + cu_k = torch.zeros(B + 1, device=q.device, dtype=torch.int32) + cu_k[1:] = torch.cumsum(klens, 0).to(torch.int32) + q_packed = q.reshape(B * Sq, Hq, D) + if paged: + k_packed, v_packed = k_cache, v_cache # kernel gathers via block_table + else: + k_packed = torch.cat([k_cache[b, : int(klens[b])] for b in range(B)], dim=0) + v_packed = torch.cat([v_cache[b, : int(klens[b])] for b in range(B)], dim=0) + + out = flash_attn_varlen_func( + q_packed, k_packed, v_packed, cu_q, cu_k, Sq, int(klens.max().item()), + softmax_scale=softmax_scale, causal=causal, + window_size=(window_size_left, window_size_right, sink_size), + block_table=block_table, s_aux=s_aux, return_lse=return_lse, + ) + if return_lse: + out, lse = out + return out.reshape(B, Sq, Hq, D), lse + return out.reshape(B, Sq, Hq, D) diff --git a/build/torch214-rocm714-x86_64-linux/symbols.json b/build/torch214-rocm714-x86_64-linux/symbols.json new file mode 100644 index 0000000000000000000000000000000000000000..e8348db4bd94712352e7fb3e83b1d94f47c2814b --- /dev/null +++ b/build/torch214-rocm714-x86_64-linux/symbols.json @@ -0,0 +1,349 @@ +{ + "schema_version": 1, + "module": "_aiter_flash_attn_ck_rocm_fe2bb66", + "functions": [ + { + "name": "flash_attn_func", + "qualname": "flash_attn_func", + "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha", + "kind": "function", + "doc": "Dense FlashAttention forward.\n\n``q, k, v`` have shape ``(batch, seqlen, nheads, headdim)``. ``window_size``\nis ``(left, right, sink_size)``; ``s_aux`` is an optional ``(nheads_q,)``\nfp32 tensor of learnable sink logits. Extra keyword arguments (e.g. those\npassed by the transformers flash-attention interface) are ignored.", + "signature": { + "parameters": [ + { + "name": "q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "v", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "dropout_p", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "0.0", + "annotation": "float" + }, + { + "name": "softmax_scale", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "float | None" + }, + { + "name": "causal", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "window_size", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "(-1, -1, 0)", + "annotation": "Tuple[int, ...]" + }, + { + "name": "bias", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "alibi_slopes", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "return_lse", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "return_attn_probs", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "s_aux", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "kwargs", + "kind": "VAR_KEYWORD", + "default": null, + "annotation": null + } + ], + "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]" + } + }, + { + "name": "flash_attn_varlen_func", + "qualname": "flash_attn_varlen_func", + "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha", + "kind": "function", + "doc": "Variable-length / packed FlashAttention forward.\n\n``q`` has shape ``(total_q, nheads, headdim)``; ``cu_seqlens_*`` are int32\ncumulative offsets of shape ``(batch + 1,)``. Sinks are supported through\n``window_size[2]`` / ``s_aux`` as in :func:`flash_attn_func`. Extra keyword\narguments (e.g. from the transformers flash-attention interface) are ignored.", + "signature": { + "parameters": [ + { + "name": "q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "v", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "cu_seqlens_q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "cu_seqlens_k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "max_seqlen_q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "int" + }, + { + "name": "max_seqlen_k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "int" + }, + { + "name": "dropout_p", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "0.0", + "annotation": "float" + }, + { + "name": "softmax_scale", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "float | None" + }, + { + "name": "causal", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "window_size", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "(-1, -1, 0)", + "annotation": "Tuple[int, ...]" + }, + { + "name": "bias", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "alibi_slopes", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "return_lse", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "return_attn_probs", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "block_table", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "min_seqlen_q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "0", + "annotation": "int" + }, + { + "name": "logits_soft_cap", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "0.0", + "annotation": "float" + }, + { + "name": "zero_tensors", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "cu_seqlens_q_padded", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "cu_seqlens_k_padded", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "s_aux", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "kwargs", + "kind": "VAR_KEYWORD", + "default": null, + "annotation": null + } + ], + "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]" + } + }, + { + "name": "flash_attn_with_kvcache", + "qualname": "flash_attn_with_kvcache", + "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha", + "kind": "function", + "doc": "FlashAttention against a KV cache, built on the CK split-KV kernel.\n\n``q`` is ``(batch, seqlen_q, nheads, headdim)``. For a contiguous cache,\n``k_cache``/``v_cache`` are ``(batch, seqlen_cache, nheads_k, headdim)`` and\n``cache_seqlens`` (int or ``(batch,)``) is the number of valid cache entries\n*before* appending. If ``k``/``v`` (the new tokens) are given, they are\nwritten into the cache in place. The query attends to the valid prefix\n(``cache_seqlens`` + ``seqlen_q``) with bottom-right causal alignment.\n\nNotes / current limitations:\n- Rotary embedding is not applied here; pass ``q``/``k`` already rotated\n (``rotary_cos``/``rotary_sin`` are rejected).\n- Paged caches are addressed via ``block_table`` (forwarded to the split-KV\n varlen kernel); in-cache append for paged layouts must be done by the\n caller (pass ``k=v=None``).", + "signature": { + "parameters": [ + { + "name": "q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "k_cache", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "v_cache", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "v", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "rotary_cos", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "rotary_sin", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "cache_seqlens", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": null + }, + { + "name": "block_table", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "softmax_scale", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "float | None" + }, + { + "name": "causal", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "True", + "annotation": "bool" + }, + { + "name": "window_size", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "(-1, -1, 0)", + "annotation": "Tuple[int, ...]" + }, + { + "name": "s_aux", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "return_lse", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "kwargs", + "kind": "VAR_KEYWORD", + "default": null, + "annotation": null + } + ], + "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]" + } + } + ], + "layers": [] +} diff --git a/build/torch214-rocm72-x86_64-linux/__init__.py b/build/torch214-rocm72-x86_64-linux/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..73acb58c1142a1ab2a6ed9d6f34bc6a53ab5e9b5 --- /dev/null +++ b/build/torch214-rocm72-x86_64-linux/__init__.py @@ -0,0 +1,27 @@ +"""AITER Composable-Kernel FlashAttention kernels for AMD ROCm (compiled HIP). + +Compiled (Composable Kernel / HIP) counterpart to the Triton-based +``aiter-flash-attn`` package, repackaged from the CK FMHA forward path of the +`ROCm/aiter `_ project as a self-contained +Hugging Face Hub kernel. Exposes the FlashAttention forward entry points with +learnable attention sink support (e.g. gpt-oss). +""" + +from .mha import ( + flash_attn_func, + flash_attn_varlen_func, + flash_attn_with_kvcache, +) + + +__kernel_metadata__ = { + "license": "mit", +} + + +__all__ = [ + "__kernel_metadata__", + "flash_attn_func", + "flash_attn_varlen_func", + "flash_attn_with_kvcache", +] diff --git a/build/torch214-rocm72-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so b/build/torch214-rocm72-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so new file mode 100644 index 0000000000000000000000000000000000000000..df749f894555f4922839a4513907fb6ac120c314 --- /dev/null +++ b/build/torch214-rocm72-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:70112e58b5f6f305e179d02f8ea407d98ec5a3bd6334e0f80a37ce2b39c43775 +size 189901912 diff --git a/build/torch214-rocm72-x86_64-linux/_ops.py b/build/torch214-rocm72-x86_64-linux/_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..2b4dd56c220b61fd2c16e0c1a02ee91a491cef0c --- /dev/null +++ b/build/torch214-rocm72-x86_64-linux/_ops.py @@ -0,0 +1,9 @@ +import torch +from . import _aiter_flash_attn_ck_rocm_fe2bb66 +ops = torch.ops._aiter_flash_attn_ck_rocm_fe2bb66 + +def add_op_namespace_prefix(op_name: str): + """ + Prefix op by namespace. + """ + return f"_aiter_flash_attn_ck_rocm_fe2bb66::{op_name}" diff --git a/build/torch214-rocm72-x86_64-linux/metadata.json b/build/torch214-rocm72-x86_64-linux/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..0ed56b558d1b9231d0ebddc4af717f67f3943edb --- /dev/null +++ b/build/torch214-rocm72-x86_64-linux/metadata.json @@ -0,0 +1,37 @@ +{ + "name": "aiter-flash-attn-ck", + "id": "_aiter_flash_attn_ck_rocm_fe2bb66", + "version": 1, + "kernels-minver": "0.14.0", + "license": "MIT", + "python-depends": [], + "kernel-depends": [], + "backend": { + "type": "rocm", + "archs": [ + "gfx942", + "gfx950" + ] + }, + "digest": { + "algorithm": "sha256", + "files": { + "__init__.py": "7wj1GTm2bqTLLCyT4xLuAk/+JWYrhINjY/ODMNWhgZ0=", + "_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so": "cBEuWLX28wXhedAvjqQH2Y7Fo71jNOD4CjfOKznEN3U=", + "_ops.py": "4WkdYDWWJp60zep+oW/5ym9vIeVMNfrgRCkP3o/w7yU=", + "mha.py": "zCAjY4O+DiCzmEL+W9sqXxZaqjsj4RJbWFcyPpUZejk=", + "symbols.json": "rhnF4H8Jsb6I+YPxIVFMgnLcTQJODEXbZ7ge77NZXAU=" + } + }, + "provenance": { + "kernel-builder": { + "version": "0.18.0-dev0", + "commit": "d054906a13710b811b96697b4635f03020842733", + "dirty": false + }, + "kernel": { + "commit": "fe2bb66fee6c2a77eccf08a0508dc57428d488ed", + "dirty": false + } + } +} \ No newline at end of file diff --git a/build/torch214-rocm72-x86_64-linux/metadata.json.sigstore b/build/torch214-rocm72-x86_64-linux/metadata.json.sigstore new file mode 100644 index 0000000000000000000000000000000000000000..6aaa91921f4512d9ff2cc24ac8503c9a84dd44a1 --- /dev/null +++ b/build/torch214-rocm72-x86_64-linux/metadata.json.sigstore @@ -0,0 +1 @@ +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json", "verificationMaterial":{"certificate":{"rawBytes":"MIIHLDCCBrGgAwIBAgIUXU4H0AQA3xJWZvqh7Y5My9WhobMwCgYIKoZIzj0EAwMwNzEVMBMGA1UEChMMc2lnc3RvcmUuZGV2MR4wHAYDVQQDExVzaWdzdG9yZS1pbnRlcm1lZGlhdGUwHhcNMjYxMDE5MjIzMTIxWhcNMjYxMDE5MjI0MTIxWjAAMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE+1g2qaLfkd0wvlA7H22LodrWcNWGe1sJkzvm2iF8/QsurJEGYtsl/+KoZuzeGuoNnj2LwnyAUABaSf2+Jfy8oqOCBdAwggXMMA4GA1UdDwEB/wQEAwIHgDATBgNVHSUEDDAKBggrBgEFBQcDAzAfBgNVHSMEGDAWgBTf0+nPViQRlvmo2OkoVaLGLhhkPzBrBgNVHREBAf8EYTBfhl1odHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkvLmdpdGh1Yi93b3JrZmxvd3MvYnVpbGQueWFtbEByZWZzL2hlYWRzL21haW4wOQYKKwYBBAGDvzABAQQraHR0cHM6Ly90b2tlbi5hY3Rpb25zLmdpdGh1YnVzZXJjb250ZW50LmNvbTAfBgorBgEEAYO/MAECBBF3b3JrZmxvd19kaXNwYXRjaDA2BgorBgEEAYO/MAEDBChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMBMGCisGAQQBg78wAQQEBUJ1aWxkMCsGCisGAQQBg78wAQUEHWh1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5MB0GCisGAQQBg78wAQYED3JlZnMvaGVhZHMvbWFpbjA7BgorBgEEAYO/MAEIBC0MK2h0dHBzOi8vdG9rZW4uYWN0aW9ucy5naXRodWJ1c2VyY29udGVudC5jb20wbQYKKwYBBAGDvzABCQRfDF1odHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkvLmdpdGh1Yi93b3JrZmxvd3MvYnVpbGQueWFtbEByZWZzL2hlYWRzL21haW4wOAYKKwYBBAGDvzABCgQqDChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMBsGCisGAQQBg78wAQsEDQwLc2VsZi1ob3N0ZWQwQAYKKwYBBAGDvzABDAQyDDBodHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkwOAYKKwYBBAGDvzABDQQqDChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMB8GCisGAQQBg78wAQ4EEQwPcmVmcy9oZWFkcy9tYWluMBoGCisGAQQBg78wAQ8EDAwKMTA3MTQ3NTUyOTAuBgorBgEEAYO/MAEQBCAMHmh0dHBzOi8vZ2l0aHViLmNvbS9odWdnaW5nZmFjZTAYBgorBgEEAYO/MAERBAoMCDI1NzIwNzQzMG0GCisGAQQBg78wARIEXwxdaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5Ly5naXRodWIvd29ya2Zsb3dzL2J1aWxkLnlhbWxAcmVmcy9oZWFkcy9tYWluMDgGCisGAQQBg78wARMEKgwoZmUyYmI2NmZlZTZjMmE3N2VjY2YwOGEwNTA4ZGM1NzQyOGQ0ODhlZDAhBgorBgEEAYO/MAEUBBMMEXdvcmtmbG93X2Rpc3BhdGNoMGQGCisGAQQBg78wARUEVgxUaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5L2FjdGlvbnMvcnVucy8zOTIzOTUxODEwNC9hdHRlbXB0cy8xMBYGCisGAQQBg78wARYECAwGcHVibGljMEYGCisGAQQBg78wARgEOAw2cmVwbzpodWdnaW5nZmFjZS9rZXJuZWxzLWNvbW11bml0eTpyZWY6cmVmcy9oZWFkcy9tYWluMIGJBgorBgEEAdZ5AgQCBHsEeQB3AHUA3T0wasbHETJjGR4cmWc3AqJKXrjePK3/h4pygC8p7o4AAAGhVkpyNQAABAMARjBEAiBuWU7juxpT5rXG8vyQ3621KaJGedJ3+9WqLubBEid8IAIgauha1RQNyucu5mlcSYjvjLnfkv6sbMqfhfmgq/M1ZBYwCgYIKoZIzj0EAwMDaQAwZgIxAPMz/yG2pxfL0/qTDpq7hjUW2u5vNixFtQeyJBLeZDUZ5By+iJ8IN45kdwX2aaP0aQIxAN+mkrm1ahVDjgDSJIUIAZZrMVIw5XgR+Bao8tzEx6ItIlyOLxtiV/CKGp2AK+KnEA=="}, "tlogEntries":[{"logIndex":"3327212768", "logId":{"keyId":"wNI9atQGlz+VWfO6LRygH4QUfY/8W4RFwiT5i5WRgB0="}, "kindVersion":{"kind":"hashedrekord", "version":"0.0.1"}, "integratedTime":"1792449082", "inclusionPromise":{"signedEntryTimestamp":"MEYCIQDMGMm5/ca/SPmAsu7KxCWMUo4Hxlk6KRXqKEHtdXoQSwIhAK2ARST2xg4MnZs5uco8+5A6pvahKaAOWe9JtYUYkqJT"}, "inclusionProof":{"logIndex":"3205308506", "rootHash":"xlBZ0fp8FNm+ElgUrRHdcr8R1Y8j0gsabT2a/uBYV/E=", "treeSize":"3205308555", "hashes":["grdKP/s7LwScwMRskvjtPWQM3J7tJGf2kHWrBQuBEBw=", "A3kzUiHiu5w6scjdbrApsAo21rb40zigFu77S4nbtx4=", "JwhQ0YwtHgan4u++foLa/2tqCPVWT6r/V2zYjcBRf6Y=", "qYHUwg5KXOhes/s3pw1j2t3+Hzyo+RgmsI24mr5aZDw=", "rSVqHYUk0OfwoqMcOFLXZv97QPjXkKJdvsk/j1rl6Hk=", "RC1UylsxdatPXrHTYD17N1c8eYHtLRRkzL+DfeVIWDw=", "6Et4r3A16F2nrBfY1l0QQlY0iHVmQlh431KkoH6BnOA=", "4/11U2tgOuRvgHTObn1t+4cs+nx+5UGraOUCW0zxSoU=", "1xxYV6+TE9ji17A5a8fHUMJY41tBhqzb3fUw9AcY6/M=", "QIMkwQbwAewhLJd//5FFwI3SnyNQBGjCgJrelfG8vhc=", "Et17CACUv9QN64irUJuBjsrgRGkGnnJ9xacRV/aBdwQ=", "lqgEwkxhDMs/gdIAE4lmjfMwg7nDmv7lda6O0LGliCw=", "HovTPBWzoErvUg1u5sIi3x1ZonZ7yJekVsBrmpomCIA=", "G+nk7mgE5h6dUmv2UY7NDJrY3WvrZY31raaO1AETt+o=", "qlTWNDzERnTBQVcHHXXVPjiUP0SsikGxfcxVlgqVFDo=", "gg9GNApNke3+zNhl4tsWtmIycCFrOMCB3lys6XuSQNQ=", "NlOclldmKPA1tMKIskqHAwuQDYHp31YwpK+OOMrMccc=", "qxzHanAzz57SDdmJe0B7bJK72NTIBbwEMGBBKvDOROw=", "xH/DCseLHr9eKoYT8qsORZK7zVdEGYWHuVtsVrD95wY="], "checkpoint":{"envelope":"rekor.sigstore.dev - 1193050959916656506\n3205308555\nxlBZ0fp8FNm+ElgUrRHdcr8R1Y8j0gsabT2a/uBYV/E=\n\n— rekor.sigstore.dev wNI9ajBEAiAmcqbuh1UcNImMs+uS4tNc9UeGREou1OqU9a409+6VhAIgYpgG3u3MKD8vQ864Rei2MpIwrgyXVPSCgMD/HU8vhN8=\n"}}, "canonicalizedBody":"eyJhcGlWZXJzaW9uIjoiMC4wLjEiLCJraW5kIjoiaGFzaGVkcmVrb3JkIiwic3BlYyI6eyJkYXRhIjp7Imhhc2giOnsiYWxnb3JpdGhtIjoic2hhMjU2IiwidmFsdWUiOiIxYzFjNTMxNzY0ODhiOWEwOWQ5NmM5NDI4NjQxNTFlNDA0YzQ5OTYxN2EyMmY1ZmM2YzExNzVlNDMwOWY3ZmU3In19LCJzaWduYXR1cmUiOnsiY29udGVudCI6Ik1FWUNJUURKcVVKYUJqdVFEbDBOdGRac1lPOFNFRWJ2STYraUZpc3BhZFF5K1dIRlBnSWhBTi85ZTQvM1YwQUxsWjJLL2Z6VlhrakxsbGliam5CS3ZVaXczdkVUQ1c0MiIsInB1YmxpY0tleSI6eyJjb250ZW50IjoiTFMwdExTMUNSVWRKVGlCRFJWSlVTVVpKUTBGVVJTMHRMUzB0Q2sxSlNVaE1SRU5EUW5KSFowRjNTVUpCWjBsVldGVTBTREJCVVVFemVFcFhXblp4YURkWk5VMTVPVmRvYjJKTmQwTm5XVWxMYjFwSmVtb3dSVUYzVFhjS1RucEZWazFDVFVkQk1WVkZRMmhOVFdNeWJHNWpNMUoyWTIxVmRWcEhWakpOVWpSM1NFRlpSRlpSVVVSRmVGWjZZVmRrZW1SSE9YbGFVekZ3WW01U2JBcGpiVEZzV2tkc2FHUkhWWGRJYUdOT1RXcFplRTFFUlRWTmFrbDZUVlJKZUZkb1kwNU5hbGw0VFVSRk5VMXFTVEJOVkVsNFYycEJRVTFHYTNkRmQxbElDa3R2V2tsNmFqQkRRVkZaU1V0dldrbDZhakJFUVZGalJGRm5RVVVyTVdjeWNXRk1abXRrTUhkMmJFRTNTREl5VEc5a2NsZGpUbGRIWlRGelNtdDZkbTBLTW1sR09DOVJjM1Z5U2tWSFdYUnpiQzhyUzI5YWRYcGxSM1Z2VG01cU1reDNibmxCVlVGQ1lWTm1NaXRLWm5rNGIzRlBRMEprUVhkbloxaE5UVUUwUndwQk1WVmtSSGRGUWk5M1VVVkJkMGxJWjBSQlZFSm5UbFpJVTFWRlJFUkJTMEpuWjNKQ1owVkdRbEZqUkVGNlFXWkNaMDVXU0ZOTlJVZEVRVmRuUWxSbUNqQXJibEJXYVZGU2JIWnRiekpQYTI5V1lVeEhUR2hvYTFCNlFuSkNaMDVXU0ZKRlFrRm1PRVZaVkVKbWFHd3hiMlJJVW5kamVtOTJUREprY0dSSGFERUtXV2sxYW1JeU1IWmhTRlp1V2pKc2RWb3lXbWhaTWxWMllUSldlV0p0Vm5OamVURnFZakl4ZEdSWE5YQmtTR3QyVEcxa2NHUkhhREZaYVRrellqTktjZ3BhYlhoMlpETk5kbGx1Vm5CaVIxRjFaVmRHZEdKRlFubGFWMXA2VERKb2JGbFhVbnBNTWpGb1lWYzBkMDlSV1V0TGQxbENRa0ZIUkhaNlFVSkJVVkZ5Q21GSVVqQmpTRTAyVEhrNU1HSXlkR3hpYVRWb1dUTlNjR0l5TlhwTWJXUndaRWRvTVZsdVZucGFXRXBxWWpJMU1GcFhOVEJNYlU1MllsUkJaa0puYjNJS1FtZEZSVUZaVHk5TlFVVkRRa0pHTTJJelNuSmFiWGgyWkRFNWEyRllUbmRaV0ZKcVlVUkJNa0puYjNKQ1owVkZRVmxQTDAxQlJVUkNRMmh0V2xSS2FRcFphbGt5V20xV2JFNXRUWGxaVkdNeldsZE9hbHBxUVRSWlZFRXhUVVJvYTFsNlZUTk9SRWswV2tSUk5FOUhWbXROUWsxSFEybHpSMEZSVVVKbk56aDNDa0ZSVVVWQ1ZVb3hZVmQ0YTAxRGMwZERhWE5IUVZGUlFtYzNPSGRCVVZWRlNGZG9NVm95WkhCaWJXUnRXVmRPYkV3eWRHeGpiVFZzWWtoTmRGa3lPWFFLWWxoV2RXRllValZOUWpCSFEybHpSMEZSVVVKbk56aDNRVkZaUlVRelNteGFiazEyWVVkV2FGcElUWFppVjBad1ltcEJOMEpuYjNKQ1owVkZRVmxQTHdwTlFVVkpRa013VFVzeWFEQmtTRUo2VDJrNGRtUkhPWEphVnpSMVdWZE9NR0ZYT1hWamVUVnVZVmhTYjJSWFNqRmpNbFo1V1RJNWRXUkhWblZrUXpWcUNtSXlNSGRpVVZsTFMzZFpRa0pCUjBSMmVrRkNRMUZTWmtSR01XOWtTRkozWTNwdmRrd3laSEJrUjJneFdXazFhbUl5TUhaaFNGWnVXakpzZFZveVdtZ0tXVEpWZG1FeVZubGliVlp6WTNreGFtSXlNWFJrVnpWd1pFaHJka3h0WkhCa1IyZ3hXV2s1TTJJelNuSmFiWGgyWkROTmRsbHVWbkJpUjFGMVpWZEdkQXBpUlVKNVdsZGFla3d5YUd4WlYxSjZUREl4YUdGWE5IZFBRVmxMUzNkWlFrSkJSMFIyZWtGQ1EyZFJjVVJEYUcxYVZFcHBXV3BaTWxwdFZteE9iVTE1Q2xsVVl6TmFWMDVxV21wQk5GbFVRVEZOUkdocldYcFZNMDVFU1RSYVJGRTBUMGRXYTAxQ2MwZERhWE5IUVZGUlFtYzNPSGRCVVhORlJGRjNUR015Vm5NS1dta3hiMkl6VGpCYVYxRjNVVUZaUzB0M1dVSkNRVWRFZG5wQlFrUkJVWGxFUkVKdlpFaFNkMk42YjNaTU1tUndaRWRvTVZscE5XcGlNakIyWVVoV2JncGFNbXgxV2pKYWFGa3lWWFpoTWxaNVltMVdjMk41TVdwaU1qRjBaRmMxY0dSSWEzZFBRVmxMUzNkWlFrSkJSMFIyZWtGQ1JGRlJjVVJEYUcxYVZFcHBDbGxxV1RKYWJWWnNUbTFOZVZsVVl6TmFWMDVxV21wQk5GbFVRVEZOUkdocldYcFZNMDVFU1RSYVJGRTBUMGRXYTAxQ09FZERhWE5IUVZGUlFtYzNPSGNLUVZFMFJVVlJkMUJqYlZadFkzazViMXBYUm10amVUbDBXVmRzZFUxQ2IwZERhWE5IUVZGUlFtYzNPSGRCVVRoRlJFRjNTMDFVUVROTlZGRXpUbFJWZVFwUFZFRjFRbWR2Y2tKblJVVkJXVTh2VFVGRlVVSkRRVTFJYldnd1pFaENlazlwT0haYU1td3dZVWhXYVV4dFRuWmlVemx2WkZka2JtRlhOVzVhYlVacUNscFVRVmxDWjI5eVFtZEZSVUZaVHk5TlFVVlNRa0Z2VFVORVNURk9la2wzVG5wUmVrMUhNRWREYVhOSFFWRlJRbWMzT0hkQlVrbEZXSGQ0WkdGSVVqQUtZMGhOTmt4NU9XNWhXRkp2WkZkSmRWa3lPWFJNTW1neFdqSmtjR0p0WkcxWlYwNXNUREowYkdOdE5XeGlTRTEwV1RJNWRHSllWblZoV0ZJMVRIazFiZ3BoV0ZKdlpGZEpkbVF5T1hsaE1scHpZak5rZWt3eVNqRmhWM2hyVEc1c2FHSlhlRUZqYlZadFkzazViMXBYUm10amVUbDBXVmRzZFUxRVowZERhWE5IQ2tGUlVVSm5OemgzUVZKTlJVdG5kMjlhYlZWNVdXMUpNazV0V214YVZGcHFUVzFGTTA0eVZtcFpNbGwzVDBkRmQwNVVRVFJhUjAweFRucFJlVTlIVVRBS1QwUm9iRnBFUVdoQ1oyOXlRbWRGUlVGWlR5OU5RVVZWUWtKTlRVVllaSFpqYlhSdFlrYzVNMWd5VW5Cak0wSm9aRWRPYjAxSFVVZERhWE5IUVZGUlFncG5OemgzUVZKVlJWWm5lRlZoU0ZJd1kwaE5Oa3g1T1c1aFdGSnZaRmRKZFZreU9YUk1NbWd4V2pKa2NHSnRaRzFaVjA1c1RESjBiR050Tld4aVNFMTBDbGt5T1hSaVdGWjFZVmhTTlV3eVJtcGtSMngyWW01TmRtTnVWblZqZVRoNlQxUkplazlVVlhoUFJFVjNUa001YUdSSVVteGlXRUl3WTNrNGVFMUNXVWNLUTJselIwRlJVVUpuTnpoM1FWSlpSVU5CZDBkalNGWnBZa2RzYWsxRldVZERhWE5IUVZGUlFtYzNPSGRCVW1kRlQwRjNNbU50Vm5kaWVuQnZaRmRrYmdwaFZ6VnVXbTFHYWxwVE9YSmFXRXAxV2xkNGVreFhUblppVnpFeFltMXNNR1ZVY0hsYVYxazJZMjFXYldONU9XOWFWMFpyWTNrNWRGbFhiSFZOU1VkS0NrSm5iM0pDWjBWRlFXUmFOVUZuVVVOQ1NITkZaVkZDTTBGSVZVRXpWREIzWVhOaVNFVlVTbXBIVWpSamJWZGpNMEZ4U2t0WWNtcGxVRXN6TDJnMGNIa0taME00Y0Rkdk5FRkJRVWRvVm10d2VVNVJRVUZDUVUxQlVtcENSVUZwUW5WWFZUZHFkWGh3VkRWeVdFYzRkbmxSTXpZeU1VdGhTa2RsWkVvekt6bFhjUXBNZFdKQ1JXbGtPRWxCU1dkaGRXaGhNVkpSVG5sMVkzVTFiV3hqVTFscWRtcE1ibVpyZGpaellrMXhabWhtYldkeEwwMHhXa0paZDBObldVbExiMXBKQ25wcU1FVkJkMDFFWVZGQmQxcG5TWGhCVUUxNkwzbEhNbkI0Wmt3d0wzRlVSSEJ4TjJocVZWY3lkVFYyVG1sNFJuUlJaWGxLUWt4bFdrUlZXalZDZVNzS2FVbzRTVTQwTld0a2QxZ3lZV0ZRTUdGUlNYaEJUaXR0YTNKdE1XRm9Wa1JxWjBSVFNrbFZTVUZhV25KTlZrbDNOVmhuVWl0Q1lXODRkSHBGZURaSmRBcEpiSGxQVEhoMGFWWXZRMHRIY0RKQlN5dExia1ZCUFQwS0xTMHRMUzFGVGtRZ1EwVlNWRWxHU1VOQlZFVXRMUzB0TFFvPSJ9fX19"}], "timestampVerificationData":{"rfc3161Timestamps":[{"signedTimestamp":"MIICyTADAgEAMIICwAYJKoZIhvcNAQcCoIICsTCCAq0CAQMxDTALBglghkgBZQMEAgEwgbgGCyqGSIb3DQEJEAEEoIGoBIGlMIGiAgEBBgkrBgEEAYO/MAIwMTANBglghkgBZQMEAgEFAAQgijHbvtFGV+0CusWKHVw0Z7h6lJV+AtW/ImB/jfYSdg0CFQCIETQefhv0SGEz5qM+71BQ1naWPBgPMjAyNjEwMTkyMjMxMjJaMAMCAQGgMqQwMC4xFTATBgNVBAoTDHNpZ3N0b3JlLmRldjEVMBMGA1UEAxMMc2lnc3RvcmUtdHNhoAAxggHaMIIB1gIBATBRMDkxFTATBgNVBAoTDHNpZ3N0b3JlLmRldjEgMB4GA1UEAxMXc2lnc3RvcmUtdHNhLXNlbGZzaWduZWQCFDoTVC8MkGHuvMFDL8uKjosqI4sMMAsGCWCGSAFlAwQCAaCB/DAaBgkqhkiG9w0BCQMxDQYLKoZIhvcNAQkQAQQwHAYJKoZIhvcNAQkFMQ8XDTI2MTAxOTIyMzEyMlowLwYJKoZIhvcNAQkEMSIEIBmQVOs/8d9ssKSicjlc4mrg61fUUMnd46O1ZcozrDtDMIGOBgsqhkiG9w0BCRACLzF/MH0wezB5BCCF+Se8B6tiysO0Q1bBDvyBssaIP9p6uebYcNnROs0FtzBVMD2kOzA5MRUwEwYDVQQKEwxzaWdzdG9yZS5kZXYxIDAeBgNVBAMTF3NpZ3N0b3JlLXRzYS1zZWxmc2lnbmVkAhQ6E1QvDJBh7rzBQy/Lio6LKiOLDDAKBggqhkjOPQQDAgRmMGQCMEtozfHyt/5Wc2+ZUl8ksXPcjLchux63Unw9AynsPvXTPuDB8Z1R72aExRKCak8lvgIwOBteBSOSoUKX7HSX8yREnOhGQCourV+2DW3VMjZ4RC1oC/upyyeKK0tq921MtEHZ"}]}}, "messageSignature":{"messageDigest":{"algorithm":"SHA2_256", "digest":"HBxTF2SIuaCdlslChkFR5ATEmWF6IvX8bBF15DCff+c="}, "signature":"MEYCIQDJqUJaBjuQDl0NtdZsYO8SEEbvI6+iFispadQy+WHFPgIhAN/9e4/3V0ALlZ2K/fzVXkjLllibjnBKvUiw3vETCW42"}} \ No newline at end of file diff --git a/build/torch214-rocm72-x86_64-linux/mha.py b/build/torch214-rocm72-x86_64-linux/mha.py new file mode 100644 index 0000000000000000000000000000000000000000..01e2d6a0fbee98d27129f2eb695ac38df73e6487 --- /dev/null +++ b/build/torch214-rocm72-x86_64-linux/mha.py @@ -0,0 +1,332 @@ +"""FlashAttention forward entry points backed by aiter's Composable-Kernel FMHA. + +This is the compiled (HIP / Composable Kernel) counterpart to the Triton +``aiter-flash-attn`` package. Only the *forward* pass is provided (dense and +variable-length), including support for learnable attention sinks via +``window_size[2]`` / ``s_aux`` (e.g. gpt-oss). The op set ships exclusively +the CK ``mha_fwd`` / ``mha_varlen_fwd`` kernels, so the sink path is always +taken (no fall-through to a non-sink ASM kernel). +""" + +from typing import Optional, Tuple, Union + +import torch +from torch import Tensor + +from ._ops import ops + + +def _round_multiple(x: int, m: int) -> int: + return (x + m - 1) // m * m + + +def _maybe_pad_headdim(t: Tensor, target: int) -> Tensor: + og = t.size(-1) + if og == target: + return t + return torch.nn.functional.pad(t, [0, target - og]) + + +def _unpack_window(window_size) -> Tuple[int, int, int]: + left = int(window_size[0]) + right = int(window_size[1]) + sink = int(window_size[2]) if len(window_size) == 3 else 0 + return left, right, sink + + +def _prep_sink(s_aux: Optional[Tensor], nheads: int, device) -> Optional[Tensor]: + if s_aux is None: + return None + assert s_aux.device == device, "s_aux must be on the same device as q" + assert s_aux.shape[0] == nheads, "s_aux must have shape [nheads_q]" + if s_aux.dtype != torch.float32: + s_aux = s_aux.to(torch.float32) + return s_aux + + +def _effective_sink_size(sink_size: int, s_aux: Optional[Tensor]) -> int: + # The CK kernel only takes the sink path when sink_size > 0. Callers using + # the transformers attention interface pass the per-head sink logits as + # ``s_aux`` but leave ``window_size`` a 2-tuple (sink_size == 0), so enable + # a single learnable sink automatically when s_aux is given. + if s_aux is not None and sink_size == 0: + return 1 + return sink_size + + +def flash_attn_func( + q: Tensor, + k: Tensor, + v: Tensor, + dropout_p: float = 0.0, + softmax_scale: Optional[float] = None, + causal: bool = False, + window_size: Tuple[int, ...] = (-1, -1, 0), + bias: Optional[Tensor] = None, + alibi_slopes: Optional[Tensor] = None, + return_lse: bool = False, + return_attn_probs: bool = False, + s_aux: Optional[Tensor] = None, + **kwargs, +) -> Union[Tensor, Tuple[Tensor, ...]]: + """Dense FlashAttention forward. + + ``q, k, v`` have shape ``(batch, seqlen, nheads, headdim)``. ``window_size`` + is ``(left, right, sink_size)``; ``s_aux`` is an optional ``(nheads_q,)`` + fp32 tensor of learnable sink logits. Extra keyword arguments (e.g. those + passed by the transformers flash-attention interface) are ignored. + """ + if softmax_scale is None: + softmax_scale = q.shape[-1] ** (-0.5) + window_size_left, window_size_right, sink_size = _unpack_window(window_size) + + nheads_q = q.size(2) + s_aux = _prep_sink(s_aux, nheads_q, q.device) + sink_size = _effective_sink_size(sink_size, s_aux) + + head_q_og = q.size(-1) + head_v_og = v.size(-1) + head_q = _round_multiple(head_q_og, 8) + head_v = _round_multiple(head_v_og, 8) + q = _maybe_pad_headdim(q, head_q) + k = _maybe_pad_headdim(k, head_q) + v = _maybe_pad_headdim(v, head_v) + + return_softmax = return_attn_probs and dropout_p > 0 + out, softmax_lse, S_dmask, _ = ops.mha_fwd( + q, + k, + v, + dropout_p, + softmax_scale, + causal, + window_size_left, + window_size_right, + sink_size, + return_lse or return_attn_probs, + return_softmax, + None, # cu_seqlens_q + None, # cu_seqlens_kv + None, # out + bias, + alibi_slopes, + None, # q_descale + None, # k_descale + None, # v_descale + s_aux, + None, # gen + ) + + out = out[..., :head_v_og] + if not (return_lse or return_attn_probs): + return out + result = (out,) + if return_lse: + result = result + (softmax_lse,) + if return_attn_probs: + result = result + (S_dmask,) + return result + + +def flash_attn_varlen_func( + q: Tensor, + k: Tensor, + v: Tensor, + cu_seqlens_q: Tensor, + cu_seqlens_k: Tensor, + max_seqlen_q: int, + max_seqlen_k: int, + dropout_p: float = 0.0, + softmax_scale: Optional[float] = None, + causal: bool = False, + window_size: Tuple[int, ...] = (-1, -1, 0), + bias: Optional[Tensor] = None, + alibi_slopes: Optional[Tensor] = None, + return_lse: bool = False, + return_attn_probs: bool = False, + block_table: Optional[Tensor] = None, + min_seqlen_q: int = 0, + logits_soft_cap: float = 0.0, + zero_tensors: bool = False, + cu_seqlens_q_padded: Optional[Tensor] = None, + cu_seqlens_k_padded: Optional[Tensor] = None, + s_aux: Optional[Tensor] = None, + **kwargs, +) -> Union[Tensor, Tuple[Tensor, ...]]: + """Variable-length / packed FlashAttention forward. + + ``q`` has shape ``(total_q, nheads, headdim)``; ``cu_seqlens_*`` are int32 + cumulative offsets of shape ``(batch + 1,)``. Sinks are supported through + ``window_size[2]`` / ``s_aux`` as in :func:`flash_attn_func`. Extra keyword + arguments (e.g. from the transformers flash-attention interface) are ignored. + """ + if softmax_scale is None: + softmax_scale = q.shape[-1] ** (-0.5) + window_size_left, window_size_right, sink_size = _unpack_window(window_size) + + nheads_q = q.size(1) + s_aux = _prep_sink(s_aux, nheads_q, q.device) + sink_size = _effective_sink_size(sink_size, s_aux) + + head_q_og = q.size(-1) + head_v_og = v.size(-1) + head_q = _round_multiple(head_q_og, 8) + head_v = _round_multiple(head_v_og, 8) + q = _maybe_pad_headdim(q, head_q) + k = _maybe_pad_headdim(k, head_q) + v = _maybe_pad_headdim(v, head_v) + + return_softmax = return_attn_probs and dropout_p > 0 + out, softmax_lse, S_dmask, _ = ops.mha_varlen_fwd( + q, + k, + v, + cu_seqlens_q, + cu_seqlens_k, + max_seqlen_q, + max_seqlen_k, + min_seqlen_q, + dropout_p, + softmax_scale, + logits_soft_cap, + zero_tensors, + causal, + window_size_left, + window_size_right, + sink_size, + return_lse or return_attn_probs, + return_softmax, + None, # out + block_table, + bias, + alibi_slopes, + None, # q_descale + None, # k_descale + None, # v_descale + None, # gen + cu_seqlens_q_padded, + cu_seqlens_k_padded, + s_aux, + ) + + out = out[..., :head_v_og] + if not (return_lse or return_attn_probs): + return out + result = (out,) + if return_lse: + result = result + (softmax_lse,) + if return_attn_probs: + result = result + (S_dmask,) + return result + + +def _append_to_contiguous_cache(k_cache, v_cache, k, v, cache_seqlens, Sq): + """In-place append of new k/v into a contiguous cache at per-batch offsets.""" + B = k_cache.shape[0] + if isinstance(cache_seqlens, int): + cache_seqlens = torch.full((B,), cache_seqlens, device=k_cache.device, dtype=torch.long) + pos = cache_seqlens.view(B, 1).to(torch.long) + torch.arange(Sq, device=k_cache.device).view(1, Sq) + bidx = torch.arange(B, device=k_cache.device).view(B, 1).expand(B, Sq) + k_cache[bidx, pos] = k.to(k_cache.dtype) + v_cache[bidx, pos] = v.to(v_cache.dtype) + + +def flash_attn_with_kvcache( + q: Tensor, + k_cache: Tensor, + v_cache: Tensor, + k: Optional[Tensor] = None, + v: Optional[Tensor] = None, + rotary_cos: Optional[Tensor] = None, + rotary_sin: Optional[Tensor] = None, + cache_seqlens=None, + block_table: Optional[Tensor] = None, + softmax_scale: Optional[float] = None, + causal: bool = True, + window_size: Tuple[int, ...] = (-1, -1, 0), + s_aux: Optional[Tensor] = None, + return_lse: bool = False, + **kwargs, +) -> Union[Tensor, Tuple[Tensor, ...]]: + """FlashAttention against a KV cache, built on the CK split-KV kernel. + + ``q`` is ``(batch, seqlen_q, nheads, headdim)``. For a contiguous cache, + ``k_cache``/``v_cache`` are ``(batch, seqlen_cache, nheads_k, headdim)`` and + ``cache_seqlens`` (int or ``(batch,)``) is the number of valid cache entries + *before* appending. If ``k``/``v`` (the new tokens) are given, they are + written into the cache in place. The query attends to the valid prefix + (``cache_seqlens`` + ``seqlen_q``) with bottom-right causal alignment. + + Notes / current limitations: + - Rotary embedding is not applied here; pass ``q``/``k`` already rotated + (``rotary_cos``/``rotary_sin`` are rejected). + - Paged caches are addressed via ``block_table`` (forwarded to the split-KV + varlen kernel); in-cache append for paged layouts must be done by the + caller (pass ``k=v=None``). + """ + if rotary_cos is not None or rotary_sin is not None: + raise NotImplementedError( + "flash_attn_with_kvcache does not apply rotary; rotate q/k beforehand." + ) + if softmax_scale is None: + softmax_scale = q.shape[-1] ** (-0.5) + window_size_left, window_size_right, sink_size = _unpack_window(window_size) + B, Sq, Hq, D = q.shape + s_aux = _prep_sink(s_aux, Hq, q.device) + sink_size = _effective_sink_size(sink_size, s_aux) + + paged = block_table is not None + + # ---- append new tokens into the cache (contiguous layout only) ---- + if k is not None and not paged: + assert cache_seqlens is not None, "cache_seqlens required to append into the cache" + _append_to_contiguous_cache(k_cache, v_cache, k, v, cache_seqlens, Sq) + elif k is not None and paged: + raise NotImplementedError( + "in-cache append for paged layouts is not supported; append externally " + "and call with k=v=None." + ) + + appended = 0 if k is None else Sq + + # ---- key lengths ---- + if cache_seqlens is None: + klens = torch.full((B,), k_cache.shape[1], device=q.device, dtype=torch.long) + elif isinstance(cache_seqlens, int): + klens = torch.full((B,), cache_seqlens + appended, device=q.device, dtype=torch.long) + else: + klens = cache_seqlens.to(q.device, torch.long) + appended + + uniform = bool((klens == klens[0]).all().item()) + + # ---- fast path: contiguous cache, uniform length -> dense kernel ---- + if not paged and uniform: + L = int(klens[0].item()) + return flash_attn_func( + q, k_cache[:, :L], v_cache[:, :L], + softmax_scale=softmax_scale, causal=causal, + window_size=(window_size_left, window_size_right, sink_size), + s_aux=s_aux, return_lse=return_lse, + ) + + # ---- general path: variable cache lengths -> split-KV varlen kernel ---- + cu_q = torch.arange(0, (B + 1) * Sq, Sq, device=q.device, dtype=torch.int32) + cu_k = torch.zeros(B + 1, device=q.device, dtype=torch.int32) + cu_k[1:] = torch.cumsum(klens, 0).to(torch.int32) + q_packed = q.reshape(B * Sq, Hq, D) + if paged: + k_packed, v_packed = k_cache, v_cache # kernel gathers via block_table + else: + k_packed = torch.cat([k_cache[b, : int(klens[b])] for b in range(B)], dim=0) + v_packed = torch.cat([v_cache[b, : int(klens[b])] for b in range(B)], dim=0) + + out = flash_attn_varlen_func( + q_packed, k_packed, v_packed, cu_q, cu_k, Sq, int(klens.max().item()), + softmax_scale=softmax_scale, causal=causal, + window_size=(window_size_left, window_size_right, sink_size), + block_table=block_table, s_aux=s_aux, return_lse=return_lse, + ) + if return_lse: + out, lse = out + return out.reshape(B, Sq, Hq, D), lse + return out.reshape(B, Sq, Hq, D) diff --git a/build/torch214-rocm72-x86_64-linux/symbols.json b/build/torch214-rocm72-x86_64-linux/symbols.json new file mode 100644 index 0000000000000000000000000000000000000000..e8348db4bd94712352e7fb3e83b1d94f47c2814b --- /dev/null +++ b/build/torch214-rocm72-x86_64-linux/symbols.json @@ -0,0 +1,349 @@ +{ + "schema_version": 1, + "module": "_aiter_flash_attn_ck_rocm_fe2bb66", + "functions": [ + { + "name": "flash_attn_func", + "qualname": "flash_attn_func", + "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha", + "kind": "function", + "doc": "Dense FlashAttention forward.\n\n``q, k, v`` have shape ``(batch, seqlen, nheads, headdim)``. ``window_size``\nis ``(left, right, sink_size)``; ``s_aux`` is an optional ``(nheads_q,)``\nfp32 tensor of learnable sink logits. Extra keyword arguments (e.g. those\npassed by the transformers flash-attention interface) are ignored.", + "signature": { + "parameters": [ + { + "name": "q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "v", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "dropout_p", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "0.0", + "annotation": "float" + }, + { + "name": "softmax_scale", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "float | None" + }, + { + "name": "causal", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "window_size", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "(-1, -1, 0)", + "annotation": "Tuple[int, ...]" + }, + { + "name": "bias", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "alibi_slopes", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "return_lse", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "return_attn_probs", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "s_aux", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "kwargs", + "kind": "VAR_KEYWORD", + "default": null, + "annotation": null + } + ], + "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]" + } + }, + { + "name": "flash_attn_varlen_func", + "qualname": "flash_attn_varlen_func", + "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha", + "kind": "function", + "doc": "Variable-length / packed FlashAttention forward.\n\n``q`` has shape ``(total_q, nheads, headdim)``; ``cu_seqlens_*`` are int32\ncumulative offsets of shape ``(batch + 1,)``. Sinks are supported through\n``window_size[2]`` / ``s_aux`` as in :func:`flash_attn_func`. Extra keyword\narguments (e.g. from the transformers flash-attention interface) are ignored.", + "signature": { + "parameters": [ + { + "name": "q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "v", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "cu_seqlens_q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "cu_seqlens_k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "max_seqlen_q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "int" + }, + { + "name": "max_seqlen_k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "int" + }, + { + "name": "dropout_p", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "0.0", + "annotation": "float" + }, + { + "name": "softmax_scale", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "float | None" + }, + { + "name": "causal", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "window_size", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "(-1, -1, 0)", + "annotation": "Tuple[int, ...]" + }, + { + "name": "bias", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "alibi_slopes", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "return_lse", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "return_attn_probs", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "block_table", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "min_seqlen_q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "0", + "annotation": "int" + }, + { + "name": "logits_soft_cap", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "0.0", + "annotation": "float" + }, + { + "name": "zero_tensors", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "cu_seqlens_q_padded", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "cu_seqlens_k_padded", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "s_aux", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "kwargs", + "kind": "VAR_KEYWORD", + "default": null, + "annotation": null + } + ], + "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]" + } + }, + { + "name": "flash_attn_with_kvcache", + "qualname": "flash_attn_with_kvcache", + "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha", + "kind": "function", + "doc": "FlashAttention against a KV cache, built on the CK split-KV kernel.\n\n``q`` is ``(batch, seqlen_q, nheads, headdim)``. For a contiguous cache,\n``k_cache``/``v_cache`` are ``(batch, seqlen_cache, nheads_k, headdim)`` and\n``cache_seqlens`` (int or ``(batch,)``) is the number of valid cache entries\n*before* appending. If ``k``/``v`` (the new tokens) are given, they are\nwritten into the cache in place. The query attends to the valid prefix\n(``cache_seqlens`` + ``seqlen_q``) with bottom-right causal alignment.\n\nNotes / current limitations:\n- Rotary embedding is not applied here; pass ``q``/``k`` already rotated\n (``rotary_cos``/``rotary_sin`` are rejected).\n- Paged caches are addressed via ``block_table`` (forwarded to the split-KV\n varlen kernel); in-cache append for paged layouts must be done by the\n caller (pass ``k=v=None``).", + "signature": { + "parameters": [ + { + "name": "q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "k_cache", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "v_cache", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "v", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "rotary_cos", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "rotary_sin", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "cache_seqlens", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": null + }, + { + "name": "block_table", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "softmax_scale", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "float | None" + }, + { + "name": "causal", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "True", + "annotation": "bool" + }, + { + "name": "window_size", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "(-1, -1, 0)", + "annotation": "Tuple[int, ...]" + }, + { + "name": "s_aux", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "return_lse", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "kwargs", + "kind": "VAR_KEYWORD", + "default": null, + "annotation": null + } + ], + "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]" + } + } + ], + "layers": [] +} diff --git a/build/torch215-rocm100-x86_64-linux/__init__.py b/build/torch215-rocm100-x86_64-linux/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..73acb58c1142a1ab2a6ed9d6f34bc6a53ab5e9b5 --- /dev/null +++ b/build/torch215-rocm100-x86_64-linux/__init__.py @@ -0,0 +1,27 @@ +"""AITER Composable-Kernel FlashAttention kernels for AMD ROCm (compiled HIP). + +Compiled (Composable Kernel / HIP) counterpart to the Triton-based +``aiter-flash-attn`` package, repackaged from the CK FMHA forward path of the +`ROCm/aiter `_ project as a self-contained +Hugging Face Hub kernel. Exposes the FlashAttention forward entry points with +learnable attention sink support (e.g. gpt-oss). +""" + +from .mha import ( + flash_attn_func, + flash_attn_varlen_func, + flash_attn_with_kvcache, +) + + +__kernel_metadata__ = { + "license": "mit", +} + + +__all__ = [ + "__kernel_metadata__", + "flash_attn_func", + "flash_attn_varlen_func", + "flash_attn_with_kvcache", +] diff --git a/build/torch215-rocm100-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so b/build/torch215-rocm100-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so new file mode 100644 index 0000000000000000000000000000000000000000..bb350f0c76f173dfe0d589f86e678454767f645c --- /dev/null +++ b/build/torch215-rocm100-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8373c8a03d6b3079d88393e5aa6153f3635d320764766e4218f094db6ed3e6ec +size 159843480 diff --git a/build/torch215-rocm100-x86_64-linux/_ops.py b/build/torch215-rocm100-x86_64-linux/_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..2b4dd56c220b61fd2c16e0c1a02ee91a491cef0c --- /dev/null +++ b/build/torch215-rocm100-x86_64-linux/_ops.py @@ -0,0 +1,9 @@ +import torch +from . import _aiter_flash_attn_ck_rocm_fe2bb66 +ops = torch.ops._aiter_flash_attn_ck_rocm_fe2bb66 + +def add_op_namespace_prefix(op_name: str): + """ + Prefix op by namespace. + """ + return f"_aiter_flash_attn_ck_rocm_fe2bb66::{op_name}" diff --git a/build/torch215-rocm100-x86_64-linux/metadata.json b/build/torch215-rocm100-x86_64-linux/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..22a56cb9a3f787f210c1cb84c33c7958246ff6cd --- /dev/null +++ b/build/torch215-rocm100-x86_64-linux/metadata.json @@ -0,0 +1,37 @@ +{ + "name": "aiter-flash-attn-ck", + "id": "_aiter_flash_attn_ck_rocm_fe2bb66", + "version": 1, + "kernels-minver": "0.14.0", + "license": "MIT", + "python-depends": [], + "kernel-depends": [], + "backend": { + "type": "rocm", + "archs": [ + "gfx942", + "gfx950" + ] + }, + "digest": { + "algorithm": "sha256", + "files": { + "__init__.py": "7wj1GTm2bqTLLCyT4xLuAk/+JWYrhINjY/ODMNWhgZ0=", + "_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so": "g3PIoD1rMHnYg5PlqmFT82NdMgdkdm5CGPCU227T5uw=", + "_ops.py": "4WkdYDWWJp60zep+oW/5ym9vIeVMNfrgRCkP3o/w7yU=", + "mha.py": "zCAjY4O+DiCzmEL+W9sqXxZaqjsj4RJbWFcyPpUZejk=", + "symbols.json": "rhnF4H8Jsb6I+YPxIVFMgnLcTQJODEXbZ7ge77NZXAU=" + } + }, + "provenance": { + "kernel-builder": { + "version": "0.18.0-dev0", + "commit": "d054906a13710b811b96697b4635f03020842733", + "dirty": false + }, + "kernel": { + "commit": "fe2bb66fee6c2a77eccf08a0508dc57428d488ed", + "dirty": false + } + } +} \ No newline at end of file diff --git a/build/torch215-rocm100-x86_64-linux/metadata.json.sigstore b/build/torch215-rocm100-x86_64-linux/metadata.json.sigstore new file mode 100644 index 0000000000000000000000000000000000000000..d9815611ab4a00d0d42a635aff2266c677a16c2f --- /dev/null +++ b/build/torch215-rocm100-x86_64-linux/metadata.json.sigstore @@ -0,0 +1 @@ +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json", "verificationMaterial":{"certificate":{"rawBytes":"MIIHLTCCBrOgAwIBAgIUCX3TUUxcAnIPEOnnGe0StlgClsEwCgYIKoZIzj0EAwMwNzEVMBMGA1UEChMMc2lnc3RvcmUuZGV2MR4wHAYDVQQDExVzaWdzdG9yZS1pbnRlcm1lZGlhdGUwHhcNMjYxMDE5MjIzMTI0WhcNMjYxMDE5MjI0MTI0WjAAMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEb+7Y/Ck/zZfLFR577T6N80he78Wet5Q7CawznF0kXJh8rMxMuE1wiFqQDMkx2FfKwzjMsILsuRg6YcyqXsIw6aOCBdIwggXOMA4GA1UdDwEB/wQEAwIHgDATBgNVHSUEDDAKBggrBgEFBQcDAzAfBgNVHSMEGDAWgBTf0+nPViQRlvmo2OkoVaLGLhhkPzBrBgNVHREBAf8EYTBfhl1odHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkvLmdpdGh1Yi93b3JrZmxvd3MvYnVpbGQueWFtbEByZWZzL2hlYWRzL21haW4wOQYKKwYBBAGDvzABAQQraHR0cHM6Ly90b2tlbi5hY3Rpb25zLmdpdGh1YnVzZXJjb250ZW50LmNvbTAfBgorBgEEAYO/MAECBBF3b3JrZmxvd19kaXNwYXRjaDA2BgorBgEEAYO/MAEDBChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMBMGCisGAQQBg78wAQQEBUJ1aWxkMCsGCisGAQQBg78wAQUEHWh1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5MB0GCisGAQQBg78wAQYED3JlZnMvaGVhZHMvbWFpbjA7BgorBgEEAYO/MAEIBC0MK2h0dHBzOi8vdG9rZW4uYWN0aW9ucy5naXRodWJ1c2VyY29udGVudC5jb20wbQYKKwYBBAGDvzABCQRfDF1odHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkvLmdpdGh1Yi93b3JrZmxvd3MvYnVpbGQueWFtbEByZWZzL2hlYWRzL21haW4wOAYKKwYBBAGDvzABCgQqDChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMBsGCisGAQQBg78wAQsEDQwLc2VsZi1ob3N0ZWQwQAYKKwYBBAGDvzABDAQyDDBodHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkwOAYKKwYBBAGDvzABDQQqDChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMB8GCisGAQQBg78wAQ4EEQwPcmVmcy9oZWFkcy9tYWluMBoGCisGAQQBg78wAQ8EDAwKMTA3MTQ3NTUyOTAuBgorBgEEAYO/MAEQBCAMHmh0dHBzOi8vZ2l0aHViLmNvbS9odWdnaW5nZmFjZTAYBgorBgEEAYO/MAERBAoMCDI1NzIwNzQzMG0GCisGAQQBg78wARIEXwxdaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5Ly5naXRodWIvd29ya2Zsb3dzL2J1aWxkLnlhbWxAcmVmcy9oZWFkcy9tYWluMDgGCisGAQQBg78wARMEKgwoZmUyYmI2NmZlZTZjMmE3N2VjY2YwOGEwNTA4ZGM1NzQyOGQ0ODhlZDAhBgorBgEEAYO/MAEUBBMMEXdvcmtmbG93X2Rpc3BhdGNoMGQGCisGAQQBg78wARUEVgxUaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5L2FjdGlvbnMvcnVucy8zOTIzOTUxODEwNC9hdHRlbXB0cy8xMBYGCisGAQQBg78wARYECAwGcHVibGljMEYGCisGAQQBg78wARgEOAw2cmVwbzpodWdnaW5nZmFjZS9rZXJuZWxzLWNvbW11bml0eTpyZWY6cmVmcy9oZWFkcy9tYWluMIGLBgorBgEEAdZ5AgQCBH0EewB5AHcA3T0wasbHETJjGR4cmWc3AqJKXrjePK3/h4pygC8p7o4AAAGhVkp6cQAABAMASDBGAiEAiWkvW+nsxp/tn2abxFLxFVWioOXu7fMmonE6/JngWIoCIQCkEGfSSyv0cjwpKTTBBi2sDVZFuNazNr98S6ekjWiiRTAKBggqhkjOPQQDAwNoADBlAjEA5Pm/DOFRd1xAnuvnYXHjVW8NHEfuky6KBmeUc0A/sIP8mRRKfp/hLvrpyA3Z79UIAjB3XIJdC1SCE/KvspODYZYgS/x6CigVwQwZXJhyAcW0kiayi3c8ZzIGUmRF2wLIlQU="}, "tlogEntries":[{"logIndex":"3327213109", "logId":{"keyId":"wNI9atQGlz+VWfO6LRygH4QUfY/8W4RFwiT5i5WRgB0="}, "kindVersion":{"kind":"hashedrekord", "version":"0.0.1"}, "integratedTime":"1792449084", "inclusionPromise":{"signedEntryTimestamp":"MEUCIQCMV0XY/Hl5msjl9hLkJk3bJkTEsra69atLCcY6m+kIvgIgQiTqSEBz0GVu5iS38pMFvVWyAjDo935R2zLqKB7mjL0="}, "inclusionProof":{"logIndex":"3205308847", "rootHash":"SSM/DC8khFBa27xhN0dSFjPfOBt7aA/sbl3EFO7/W+0=", "treeSize":"3205308870", "hashes":["55yNSanLrCPlUcmvqcbbHYzNCckZAnvue0mxmuBMc4Y=", "lB2UbQsUU1pztt/JDoFHc4pXL6TU6mB2iOxJKvShG+E=", "96DlfQ2Vn7IxKkDKXOGrijOesgFgGWUT6V4V4qhPsZw=", "F2+iZj3uo95n8S+doV4ozf1YBy6pdLhsO8YigA5+X1w=", "kPAEsko55UYQEQnhAVk10vvZtBtpl/8tKmcNgliJpGE=", "iEG8R04R6PeKhKvaNTHaAQwUwDX/SQi4XA9zZFOUvrM=", "icKwsNEXPumS0mC/eK0IEqoaRcEWbHu/tMz/5cBjr1k=", "+K+g6qNlmv/J6gWsfpY7StUOvXlltyssK8uS7s0IKFs=", "XV6tZSvlC3TqBV/+dt7tPQU+uE3xgTOUdwx2katy4Ok=", "1xxYV6+TE9ji17A5a8fHUMJY41tBhqzb3fUw9AcY6/M=", "QIMkwQbwAewhLJd//5FFwI3SnyNQBGjCgJrelfG8vhc=", "Et17CACUv9QN64irUJuBjsrgRGkGnnJ9xacRV/aBdwQ=", "lqgEwkxhDMs/gdIAE4lmjfMwg7nDmv7lda6O0LGliCw=", "HovTPBWzoErvUg1u5sIi3x1ZonZ7yJekVsBrmpomCIA=", "G+nk7mgE5h6dUmv2UY7NDJrY3WvrZY31raaO1AETt+o=", "qlTWNDzERnTBQVcHHXXVPjiUP0SsikGxfcxVlgqVFDo=", "gg9GNApNke3+zNhl4tsWtmIycCFrOMCB3lys6XuSQNQ=", "NlOclldmKPA1tMKIskqHAwuQDYHp31YwpK+OOMrMccc=", "qxzHanAzz57SDdmJe0B7bJK72NTIBbwEMGBBKvDOROw=", "xH/DCseLHr9eKoYT8qsORZK7zVdEGYWHuVtsVrD95wY="], "checkpoint":{"envelope":"rekor.sigstore.dev - 1193050959916656506\n3205308870\nSSM/DC8khFBa27xhN0dSFjPfOBt7aA/sbl3EFO7/W+0=\n\n— rekor.sigstore.dev wNI9ajBEAiAxjsMmLTxhVoxEtC5OzFh06Tr0l9Ml58U/0LwUAstdmwIgAI9F6ZoqU4g6ylGh8mQyL3COpC+gtyJiXR8JnxuTkAE=\n"}}, "canonicalizedBody":"eyJhcGlWZXJzaW9uIjoiMC4wLjEiLCJraW5kIjoiaGFzaGVkcmVrb3JkIiwic3BlYyI6eyJkYXRhIjp7Imhhc2giOnsiYWxnb3JpdGhtIjoic2hhMjU2IiwidmFsdWUiOiI5ZTNhNWUyMjE1ZmY5ODZlNDU0MzgxOTM1MWMzMzhkZWZlMGZlMjUwYWQzZGEwN2JmYTFhMDI0MjAyNWVkYTBlIn19LCJzaWduYXR1cmUiOnsiY29udGVudCI6Ik1FVUNJUUNDM0dvZWFmZ1NabCtKYUkrR05kUDlOU2Y3bHRJM0t4cGdsR1ppZnJBYTVBSWdZZUZ4QUVrWVc4OGxjYjZWOG5QcGRDWU5TMTJvTzR3b083VThvZGlkT1YwPSIsInB1YmxpY0tleSI6eyJjb250ZW50IjoiTFMwdExTMUNSVWRKVGlCRFJWSlVTVVpKUTBGVVJTMHRMUzB0Q2sxSlNVaE1WRU5EUW5KUFowRjNTVUpCWjBsVlExZ3pWRlZWZUdOQmJrbFFSVTl1YmtkbE1GTjBiR2REYkhORmQwTm5XVWxMYjFwSmVtb3dSVUYzVFhjS1RucEZWazFDVFVkQk1WVkZRMmhOVFdNeWJHNWpNMUoyWTIxVmRWcEhWakpOVWpSM1NFRlpSRlpSVVVSRmVGWjZZVmRrZW1SSE9YbGFVekZ3WW01U2JBcGpiVEZzV2tkc2FHUkhWWGRJYUdOT1RXcFplRTFFUlRWTmFrbDZUVlJKTUZkb1kwNU5hbGw0VFVSRk5VMXFTVEJOVkVrd1YycEJRVTFHYTNkRmQxbElDa3R2V2tsNmFqQkRRVkZaU1V0dldrbDZhakJFUVZGalJGRm5RVVZpS3pkWkwwTnJMM3BhWmt4R1VqVTNOMVEyVGpnd2FHVTNPRmRsZERWUk4wTmhkM29LYmtZd2ExaEthRGh5VFhoTmRVVXhkMmxHY1ZGRVRXdDRNa1ptUzNkNmFrMXpTVXh6ZFZKbk5sbGplWEZZYzBsM05tRlBRMEprU1hkbloxaFBUVUUwUndwQk1WVmtSSGRGUWk5M1VVVkJkMGxJWjBSQlZFSm5UbFpJVTFWRlJFUkJTMEpuWjNKQ1owVkdRbEZqUkVGNlFXWkNaMDVXU0ZOTlJVZEVRVmRuUWxSbUNqQXJibEJXYVZGU2JIWnRiekpQYTI5V1lVeEhUR2hvYTFCNlFuSkNaMDVXU0ZKRlFrRm1PRVZaVkVKbWFHd3hiMlJJVW5kamVtOTJUREprY0dSSGFERUtXV2sxYW1JeU1IWmhTRlp1V2pKc2RWb3lXbWhaTWxWMllUSldlV0p0Vm5OamVURnFZakl4ZEdSWE5YQmtTR3QyVEcxa2NHUkhhREZaYVRrellqTktjZ3BhYlhoMlpETk5kbGx1Vm5CaVIxRjFaVmRHZEdKRlFubGFWMXA2VERKb2JGbFhVbnBNTWpGb1lWYzBkMDlSV1V0TGQxbENRa0ZIUkhaNlFVSkJVVkZ5Q21GSVVqQmpTRTAyVEhrNU1HSXlkR3hpYVRWb1dUTlNjR0l5TlhwTWJXUndaRWRvTVZsdVZucGFXRXBxWWpJMU1GcFhOVEJNYlU1MllsUkJaa0puYjNJS1FtZEZSVUZaVHk5TlFVVkRRa0pHTTJJelNuSmFiWGgyWkRFNWEyRllUbmRaV0ZKcVlVUkJNa0puYjNKQ1owVkZRVmxQTDAxQlJVUkNRMmh0V2xSS2FRcFphbGt5V20xV2JFNXRUWGxaVkdNeldsZE9hbHBxUVRSWlZFRXhUVVJvYTFsNlZUTk9SRWswV2tSUk5FOUhWbXROUWsxSFEybHpSMEZSVVVKbk56aDNDa0ZSVVVWQ1ZVb3hZVmQ0YTAxRGMwZERhWE5IUVZGUlFtYzNPSGRCVVZWRlNGZG9NVm95WkhCaWJXUnRXVmRPYkV3eWRHeGpiVFZzWWtoTmRGa3lPWFFLWWxoV2RXRllValZOUWpCSFEybHpSMEZSVVVKbk56aDNRVkZaUlVRelNteGFiazEyWVVkV2FGcElUWFppVjBad1ltcEJOMEpuYjNKQ1owVkZRVmxQTHdwTlFVVkpRa013VFVzeWFEQmtTRUo2VDJrNGRtUkhPWEphVnpSMVdWZE9NR0ZYT1hWamVUVnVZVmhTYjJSWFNqRmpNbFo1V1RJNWRXUkhWblZrUXpWcUNtSXlNSGRpVVZsTFMzZFpRa0pCUjBSMmVrRkNRMUZTWmtSR01XOWtTRkozWTNwdmRrd3laSEJrUjJneFdXazFhbUl5TUhaaFNGWnVXakpzZFZveVdtZ0tXVEpWZG1FeVZubGliVlp6WTNreGFtSXlNWFJrVnpWd1pFaHJka3h0WkhCa1IyZ3hXV2s1TTJJelNuSmFiWGgyWkROTmRsbHVWbkJpUjFGMVpWZEdkQXBpUlVKNVdsZGFla3d5YUd4WlYxSjZUREl4YUdGWE5IZFBRVmxMUzNkWlFrSkJSMFIyZWtGQ1EyZFJjVVJEYUcxYVZFcHBXV3BaTWxwdFZteE9iVTE1Q2xsVVl6TmFWMDVxV21wQk5GbFVRVEZOUkdocldYcFZNMDVFU1RSYVJGRTBUMGRXYTAxQ2MwZERhWE5IUVZGUlFtYzNPSGRCVVhORlJGRjNUR015Vm5NS1dta3hiMkl6VGpCYVYxRjNVVUZaUzB0M1dVSkNRVWRFZG5wQlFrUkJVWGxFUkVKdlpFaFNkMk42YjNaTU1tUndaRWRvTVZscE5XcGlNakIyWVVoV2JncGFNbXgxV2pKYWFGa3lWWFpoTWxaNVltMVdjMk41TVdwaU1qRjBaRmMxY0dSSWEzZFBRVmxMUzNkWlFrSkJSMFIyZWtGQ1JGRlJjVVJEYUcxYVZFcHBDbGxxV1RKYWJWWnNUbTFOZVZsVVl6TmFWMDVxV21wQk5GbFVRVEZOUkdocldYcFZNMDVFU1RSYVJGRTBUMGRXYTAxQ09FZERhWE5IUVZGUlFtYzNPSGNLUVZFMFJVVlJkMUJqYlZadFkzazViMXBYUm10amVUbDBXVmRzZFUxQ2IwZERhWE5IUVZGUlFtYzNPSGRCVVRoRlJFRjNTMDFVUVROTlZGRXpUbFJWZVFwUFZFRjFRbWR2Y2tKblJVVkJXVTh2VFVGRlVVSkRRVTFJYldnd1pFaENlazlwT0haYU1td3dZVWhXYVV4dFRuWmlVemx2WkZka2JtRlhOVzVhYlVacUNscFVRVmxDWjI5eVFtZEZSVUZaVHk5TlFVVlNRa0Z2VFVORVNURk9la2wzVG5wUmVrMUhNRWREYVhOSFFWRlJRbWMzT0hkQlVrbEZXSGQ0WkdGSVVqQUtZMGhOTmt4NU9XNWhXRkp2WkZkSmRWa3lPWFJNTW1neFdqSmtjR0p0WkcxWlYwNXNUREowYkdOdE5XeGlTRTEwV1RJNWRHSllWblZoV0ZJMVRIazFiZ3BoV0ZKdlpGZEpkbVF5T1hsaE1scHpZak5rZWt3eVNqRmhWM2hyVEc1c2FHSlhlRUZqYlZadFkzazViMXBYUm10amVUbDBXVmRzZFUxRVowZERhWE5IQ2tGUlVVSm5OemgzUVZKTlJVdG5kMjlhYlZWNVdXMUpNazV0V214YVZGcHFUVzFGTTA0eVZtcFpNbGwzVDBkRmQwNVVRVFJhUjAweFRucFJlVTlIVVRBS1QwUm9iRnBFUVdoQ1oyOXlRbWRGUlVGWlR5OU5RVVZWUWtKTlRVVllaSFpqYlhSdFlrYzVNMWd5VW5Cak0wSm9aRWRPYjAxSFVVZERhWE5IUVZGUlFncG5OemgzUVZKVlJWWm5lRlZoU0ZJd1kwaE5Oa3g1T1c1aFdGSnZaRmRKZFZreU9YUk1NbWd4V2pKa2NHSnRaRzFaVjA1c1RESjBiR050Tld4aVNFMTBDbGt5T1hSaVdGWjFZVmhTTlV3eVJtcGtSMngyWW01TmRtTnVWblZqZVRoNlQxUkplazlVVlhoUFJFVjNUa001YUdSSVVteGlXRUl3WTNrNGVFMUNXVWNLUTJselIwRlJVVUpuTnpoM1FWSlpSVU5CZDBkalNGWnBZa2RzYWsxRldVZERhWE5IUVZGUlFtYzNPSGRCVW1kRlQwRjNNbU50Vm5kaWVuQnZaRmRrYmdwaFZ6VnVXbTFHYWxwVE9YSmFXRXAxV2xkNGVreFhUblppVnpFeFltMXNNR1ZVY0hsYVYxazJZMjFXYldONU9XOWFWMFpyWTNrNWRGbFhiSFZOU1VkTUNrSm5iM0pDWjBWRlFXUmFOVUZuVVVOQ1NEQkZaWGRDTlVGSVkwRXpWREIzWVhOaVNFVlVTbXBIVWpSamJWZGpNMEZ4U2t0WWNtcGxVRXN6TDJnMGNIa0taME00Y0Rkdk5FRkJRVWRvVm10d05tTlJRVUZDUVUxQlUwUkNSMEZwUlVGcFYydDJWeXR1YzNod0wzUnVNbUZpZUVaTWVFWldWMmx2VDFoMU4yWk5iUXB2YmtVMkwwcHVaMWRKYjBOSlVVTnJSVWRtVTFONWRqQmphbmR3UzFSVVFrSnBNbk5FVmxwR2RVNWhlazV5T1RoVE5tVnJhbGRwYVZKVVFVdENaMmR4Q21ocmFrOVFVVkZFUVhkT2IwRkVRbXhCYWtWQk5WQnRMMFJQUmxKa01YaEJiblYyYmxsWVNHcFdWemhPU0VWbWRXdDVOa3RDYldWVll6QkJMM05KVURnS2JWSlNTMlp3TDJoTWRuSndlVUV6V2pjNVZVbEJha0l6V0VsS1pFTXhVME5GTDB0MmMzQlBSRmxhV1dkVEwzZzJRMmxuVm5kUmQxcFlTbWg1UVdOWE1BcHJhV0Y1YVROak9GcDZTVWRWYlZKR01uZE1TV3hSVlQwS0xTMHRMUzFGVGtRZ1EwVlNWRWxHU1VOQlZFVXRMUzB0TFFvPSJ9fX19"}], "timestampVerificationData":{"rfc3161Timestamps":[{"signedTimestamp":"MIICyTADAgEAMIICwAYJKoZIhvcNAQcCoIICsTCCAq0CAQMxDTALBglghkgBZQMEAgEwgbgGCyqGSIb3DQEJEAEEoIGoBIGlMIGiAgEBBgkrBgEEAYO/MAIwMTANBglghkgBZQMEAgEFAAQgTD2lbIaxz4Ry8q5/WMyhCoR9bWqgwayXwDQmg8X4e4wCFQDZoTsjKdKIhkI6hvqmEyz6oHm1ZxgPMjAyNjEwMTkyMjMxMjRaMAMCAQGgMqQwMC4xFTATBgNVBAoTDHNpZ3N0b3JlLmRldjEVMBMGA1UEAxMMc2lnc3RvcmUtdHNhoAAxggHaMIIB1gIBATBRMDkxFTATBgNVBAoTDHNpZ3N0b3JlLmRldjEgMB4GA1UEAxMXc2lnc3RvcmUtdHNhLXNlbGZzaWduZWQCFDoTVC8MkGHuvMFDL8uKjosqI4sMMAsGCWCGSAFlAwQCAaCB/DAaBgkqhkiG9w0BCQMxDQYLKoZIhvcNAQkQAQQwHAYJKoZIhvcNAQkFMQ8XDTI2MTAxOTIyMzEyNFowLwYJKoZIhvcNAQkEMSIEIOCrO5EFELNntZUEfSYTccaUuxAjgimBfnjrDMFSUkBmMIGOBgsqhkiG9w0BCRACLzF/MH0wezB5BCCF+Se8B6tiysO0Q1bBDvyBssaIP9p6uebYcNnROs0FtzBVMD2kOzA5MRUwEwYDVQQKEwxzaWdzdG9yZS5kZXYxIDAeBgNVBAMTF3NpZ3N0b3JlLXRzYS1zZWxmc2lnbmVkAhQ6E1QvDJBh7rzBQy/Lio6LKiOLDDAKBggqhkjOPQQDAgRmMGQCMCpVqejvXKqxeOXaNbsjVeVC56O2rgoft2ttMzvoiMR5m2rDKLZJQUij88UI2Kb8fgIwKMU4YFk0hswV7t2a8HDORXbbsaj+qTbmvGNkADKLcJOju4X4JIJCX0KTF/f0yNiC"}]}}, "messageSignature":{"messageDigest":{"algorithm":"SHA2_256", "digest":"njpeIhX/mG5FQ4GTUcM43v4P4lCtPaB7+hoCQgJe2g4="}, "signature":"MEUCIQCC3GoeafgSZl+JaI+GNdP9NSf7ltI3KxpglGZifrAa5AIgYeFxAEkYW88lcb6V8nPpdCYNS12oO4woO7U8odidOV0="}} \ No newline at end of file diff --git a/build/torch215-rocm100-x86_64-linux/mha.py b/build/torch215-rocm100-x86_64-linux/mha.py new file mode 100644 index 0000000000000000000000000000000000000000..01e2d6a0fbee98d27129f2eb695ac38df73e6487 --- /dev/null +++ b/build/torch215-rocm100-x86_64-linux/mha.py @@ -0,0 +1,332 @@ +"""FlashAttention forward entry points backed by aiter's Composable-Kernel FMHA. + +This is the compiled (HIP / Composable Kernel) counterpart to the Triton +``aiter-flash-attn`` package. Only the *forward* pass is provided (dense and +variable-length), including support for learnable attention sinks via +``window_size[2]`` / ``s_aux`` (e.g. gpt-oss). The op set ships exclusively +the CK ``mha_fwd`` / ``mha_varlen_fwd`` kernels, so the sink path is always +taken (no fall-through to a non-sink ASM kernel). +""" + +from typing import Optional, Tuple, Union + +import torch +from torch import Tensor + +from ._ops import ops + + +def _round_multiple(x: int, m: int) -> int: + return (x + m - 1) // m * m + + +def _maybe_pad_headdim(t: Tensor, target: int) -> Tensor: + og = t.size(-1) + if og == target: + return t + return torch.nn.functional.pad(t, [0, target - og]) + + +def _unpack_window(window_size) -> Tuple[int, int, int]: + left = int(window_size[0]) + right = int(window_size[1]) + sink = int(window_size[2]) if len(window_size) == 3 else 0 + return left, right, sink + + +def _prep_sink(s_aux: Optional[Tensor], nheads: int, device) -> Optional[Tensor]: + if s_aux is None: + return None + assert s_aux.device == device, "s_aux must be on the same device as q" + assert s_aux.shape[0] == nheads, "s_aux must have shape [nheads_q]" + if s_aux.dtype != torch.float32: + s_aux = s_aux.to(torch.float32) + return s_aux + + +def _effective_sink_size(sink_size: int, s_aux: Optional[Tensor]) -> int: + # The CK kernel only takes the sink path when sink_size > 0. Callers using + # the transformers attention interface pass the per-head sink logits as + # ``s_aux`` but leave ``window_size`` a 2-tuple (sink_size == 0), so enable + # a single learnable sink automatically when s_aux is given. + if s_aux is not None and sink_size == 0: + return 1 + return sink_size + + +def flash_attn_func( + q: Tensor, + k: Tensor, + v: Tensor, + dropout_p: float = 0.0, + softmax_scale: Optional[float] = None, + causal: bool = False, + window_size: Tuple[int, ...] = (-1, -1, 0), + bias: Optional[Tensor] = None, + alibi_slopes: Optional[Tensor] = None, + return_lse: bool = False, + return_attn_probs: bool = False, + s_aux: Optional[Tensor] = None, + **kwargs, +) -> Union[Tensor, Tuple[Tensor, ...]]: + """Dense FlashAttention forward. + + ``q, k, v`` have shape ``(batch, seqlen, nheads, headdim)``. ``window_size`` + is ``(left, right, sink_size)``; ``s_aux`` is an optional ``(nheads_q,)`` + fp32 tensor of learnable sink logits. Extra keyword arguments (e.g. those + passed by the transformers flash-attention interface) are ignored. + """ + if softmax_scale is None: + softmax_scale = q.shape[-1] ** (-0.5) + window_size_left, window_size_right, sink_size = _unpack_window(window_size) + + nheads_q = q.size(2) + s_aux = _prep_sink(s_aux, nheads_q, q.device) + sink_size = _effective_sink_size(sink_size, s_aux) + + head_q_og = q.size(-1) + head_v_og = v.size(-1) + head_q = _round_multiple(head_q_og, 8) + head_v = _round_multiple(head_v_og, 8) + q = _maybe_pad_headdim(q, head_q) + k = _maybe_pad_headdim(k, head_q) + v = _maybe_pad_headdim(v, head_v) + + return_softmax = return_attn_probs and dropout_p > 0 + out, softmax_lse, S_dmask, _ = ops.mha_fwd( + q, + k, + v, + dropout_p, + softmax_scale, + causal, + window_size_left, + window_size_right, + sink_size, + return_lse or return_attn_probs, + return_softmax, + None, # cu_seqlens_q + None, # cu_seqlens_kv + None, # out + bias, + alibi_slopes, + None, # q_descale + None, # k_descale + None, # v_descale + s_aux, + None, # gen + ) + + out = out[..., :head_v_og] + if not (return_lse or return_attn_probs): + return out + result = (out,) + if return_lse: + result = result + (softmax_lse,) + if return_attn_probs: + result = result + (S_dmask,) + return result + + +def flash_attn_varlen_func( + q: Tensor, + k: Tensor, + v: Tensor, + cu_seqlens_q: Tensor, + cu_seqlens_k: Tensor, + max_seqlen_q: int, + max_seqlen_k: int, + dropout_p: float = 0.0, + softmax_scale: Optional[float] = None, + causal: bool = False, + window_size: Tuple[int, ...] = (-1, -1, 0), + bias: Optional[Tensor] = None, + alibi_slopes: Optional[Tensor] = None, + return_lse: bool = False, + return_attn_probs: bool = False, + block_table: Optional[Tensor] = None, + min_seqlen_q: int = 0, + logits_soft_cap: float = 0.0, + zero_tensors: bool = False, + cu_seqlens_q_padded: Optional[Tensor] = None, + cu_seqlens_k_padded: Optional[Tensor] = None, + s_aux: Optional[Tensor] = None, + **kwargs, +) -> Union[Tensor, Tuple[Tensor, ...]]: + """Variable-length / packed FlashAttention forward. + + ``q`` has shape ``(total_q, nheads, headdim)``; ``cu_seqlens_*`` are int32 + cumulative offsets of shape ``(batch + 1,)``. Sinks are supported through + ``window_size[2]`` / ``s_aux`` as in :func:`flash_attn_func`. Extra keyword + arguments (e.g. from the transformers flash-attention interface) are ignored. + """ + if softmax_scale is None: + softmax_scale = q.shape[-1] ** (-0.5) + window_size_left, window_size_right, sink_size = _unpack_window(window_size) + + nheads_q = q.size(1) + s_aux = _prep_sink(s_aux, nheads_q, q.device) + sink_size = _effective_sink_size(sink_size, s_aux) + + head_q_og = q.size(-1) + head_v_og = v.size(-1) + head_q = _round_multiple(head_q_og, 8) + head_v = _round_multiple(head_v_og, 8) + q = _maybe_pad_headdim(q, head_q) + k = _maybe_pad_headdim(k, head_q) + v = _maybe_pad_headdim(v, head_v) + + return_softmax = return_attn_probs and dropout_p > 0 + out, softmax_lse, S_dmask, _ = ops.mha_varlen_fwd( + q, + k, + v, + cu_seqlens_q, + cu_seqlens_k, + max_seqlen_q, + max_seqlen_k, + min_seqlen_q, + dropout_p, + softmax_scale, + logits_soft_cap, + zero_tensors, + causal, + window_size_left, + window_size_right, + sink_size, + return_lse or return_attn_probs, + return_softmax, + None, # out + block_table, + bias, + alibi_slopes, + None, # q_descale + None, # k_descale + None, # v_descale + None, # gen + cu_seqlens_q_padded, + cu_seqlens_k_padded, + s_aux, + ) + + out = out[..., :head_v_og] + if not (return_lse or return_attn_probs): + return out + result = (out,) + if return_lse: + result = result + (softmax_lse,) + if return_attn_probs: + result = result + (S_dmask,) + return result + + +def _append_to_contiguous_cache(k_cache, v_cache, k, v, cache_seqlens, Sq): + """In-place append of new k/v into a contiguous cache at per-batch offsets.""" + B = k_cache.shape[0] + if isinstance(cache_seqlens, int): + cache_seqlens = torch.full((B,), cache_seqlens, device=k_cache.device, dtype=torch.long) + pos = cache_seqlens.view(B, 1).to(torch.long) + torch.arange(Sq, device=k_cache.device).view(1, Sq) + bidx = torch.arange(B, device=k_cache.device).view(B, 1).expand(B, Sq) + k_cache[bidx, pos] = k.to(k_cache.dtype) + v_cache[bidx, pos] = v.to(v_cache.dtype) + + +def flash_attn_with_kvcache( + q: Tensor, + k_cache: Tensor, + v_cache: Tensor, + k: Optional[Tensor] = None, + v: Optional[Tensor] = None, + rotary_cos: Optional[Tensor] = None, + rotary_sin: Optional[Tensor] = None, + cache_seqlens=None, + block_table: Optional[Tensor] = None, + softmax_scale: Optional[float] = None, + causal: bool = True, + window_size: Tuple[int, ...] = (-1, -1, 0), + s_aux: Optional[Tensor] = None, + return_lse: bool = False, + **kwargs, +) -> Union[Tensor, Tuple[Tensor, ...]]: + """FlashAttention against a KV cache, built on the CK split-KV kernel. + + ``q`` is ``(batch, seqlen_q, nheads, headdim)``. For a contiguous cache, + ``k_cache``/``v_cache`` are ``(batch, seqlen_cache, nheads_k, headdim)`` and + ``cache_seqlens`` (int or ``(batch,)``) is the number of valid cache entries + *before* appending. If ``k``/``v`` (the new tokens) are given, they are + written into the cache in place. The query attends to the valid prefix + (``cache_seqlens`` + ``seqlen_q``) with bottom-right causal alignment. + + Notes / current limitations: + - Rotary embedding is not applied here; pass ``q``/``k`` already rotated + (``rotary_cos``/``rotary_sin`` are rejected). + - Paged caches are addressed via ``block_table`` (forwarded to the split-KV + varlen kernel); in-cache append for paged layouts must be done by the + caller (pass ``k=v=None``). + """ + if rotary_cos is not None or rotary_sin is not None: + raise NotImplementedError( + "flash_attn_with_kvcache does not apply rotary; rotate q/k beforehand." + ) + if softmax_scale is None: + softmax_scale = q.shape[-1] ** (-0.5) + window_size_left, window_size_right, sink_size = _unpack_window(window_size) + B, Sq, Hq, D = q.shape + s_aux = _prep_sink(s_aux, Hq, q.device) + sink_size = _effective_sink_size(sink_size, s_aux) + + paged = block_table is not None + + # ---- append new tokens into the cache (contiguous layout only) ---- + if k is not None and not paged: + assert cache_seqlens is not None, "cache_seqlens required to append into the cache" + _append_to_contiguous_cache(k_cache, v_cache, k, v, cache_seqlens, Sq) + elif k is not None and paged: + raise NotImplementedError( + "in-cache append for paged layouts is not supported; append externally " + "and call with k=v=None." + ) + + appended = 0 if k is None else Sq + + # ---- key lengths ---- + if cache_seqlens is None: + klens = torch.full((B,), k_cache.shape[1], device=q.device, dtype=torch.long) + elif isinstance(cache_seqlens, int): + klens = torch.full((B,), cache_seqlens + appended, device=q.device, dtype=torch.long) + else: + klens = cache_seqlens.to(q.device, torch.long) + appended + + uniform = bool((klens == klens[0]).all().item()) + + # ---- fast path: contiguous cache, uniform length -> dense kernel ---- + if not paged and uniform: + L = int(klens[0].item()) + return flash_attn_func( + q, k_cache[:, :L], v_cache[:, :L], + softmax_scale=softmax_scale, causal=causal, + window_size=(window_size_left, window_size_right, sink_size), + s_aux=s_aux, return_lse=return_lse, + ) + + # ---- general path: variable cache lengths -> split-KV varlen kernel ---- + cu_q = torch.arange(0, (B + 1) * Sq, Sq, device=q.device, dtype=torch.int32) + cu_k = torch.zeros(B + 1, device=q.device, dtype=torch.int32) + cu_k[1:] = torch.cumsum(klens, 0).to(torch.int32) + q_packed = q.reshape(B * Sq, Hq, D) + if paged: + k_packed, v_packed = k_cache, v_cache # kernel gathers via block_table + else: + k_packed = torch.cat([k_cache[b, : int(klens[b])] for b in range(B)], dim=0) + v_packed = torch.cat([v_cache[b, : int(klens[b])] for b in range(B)], dim=0) + + out = flash_attn_varlen_func( + q_packed, k_packed, v_packed, cu_q, cu_k, Sq, int(klens.max().item()), + softmax_scale=softmax_scale, causal=causal, + window_size=(window_size_left, window_size_right, sink_size), + block_table=block_table, s_aux=s_aux, return_lse=return_lse, + ) + if return_lse: + out, lse = out + return out.reshape(B, Sq, Hq, D), lse + return out.reshape(B, Sq, Hq, D) diff --git a/build/torch215-rocm100-x86_64-linux/symbols.json b/build/torch215-rocm100-x86_64-linux/symbols.json new file mode 100644 index 0000000000000000000000000000000000000000..e8348db4bd94712352e7fb3e83b1d94f47c2814b --- /dev/null +++ b/build/torch215-rocm100-x86_64-linux/symbols.json @@ -0,0 +1,349 @@ +{ + "schema_version": 1, + "module": "_aiter_flash_attn_ck_rocm_fe2bb66", + "functions": [ + { + "name": "flash_attn_func", + "qualname": "flash_attn_func", + "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha", + "kind": "function", + "doc": "Dense FlashAttention forward.\n\n``q, k, v`` have shape ``(batch, seqlen, nheads, headdim)``. ``window_size``\nis ``(left, right, sink_size)``; ``s_aux`` is an optional ``(nheads_q,)``\nfp32 tensor of learnable sink logits. Extra keyword arguments (e.g. those\npassed by the transformers flash-attention interface) are ignored.", + "signature": { + "parameters": [ + { + "name": "q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "v", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "dropout_p", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "0.0", + "annotation": "float" + }, + { + "name": "softmax_scale", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "float | None" + }, + { + "name": "causal", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "window_size", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "(-1, -1, 0)", + "annotation": "Tuple[int, ...]" + }, + { + "name": "bias", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "alibi_slopes", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "return_lse", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "return_attn_probs", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "s_aux", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "kwargs", + "kind": "VAR_KEYWORD", + "default": null, + "annotation": null + } + ], + "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]" + } + }, + { + "name": "flash_attn_varlen_func", + "qualname": "flash_attn_varlen_func", + "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha", + "kind": "function", + "doc": "Variable-length / packed FlashAttention forward.\n\n``q`` has shape ``(total_q, nheads, headdim)``; ``cu_seqlens_*`` are int32\ncumulative offsets of shape ``(batch + 1,)``. Sinks are supported through\n``window_size[2]`` / ``s_aux`` as in :func:`flash_attn_func`. Extra keyword\narguments (e.g. from the transformers flash-attention interface) are ignored.", + "signature": { + "parameters": [ + { + "name": "q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "v", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "cu_seqlens_q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "cu_seqlens_k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "max_seqlen_q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "int" + }, + { + "name": "max_seqlen_k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "int" + }, + { + "name": "dropout_p", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "0.0", + "annotation": "float" + }, + { + "name": "softmax_scale", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "float | None" + }, + { + "name": "causal", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "window_size", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "(-1, -1, 0)", + "annotation": "Tuple[int, ...]" + }, + { + "name": "bias", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "alibi_slopes", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "return_lse", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "return_attn_probs", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "block_table", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "min_seqlen_q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "0", + "annotation": "int" + }, + { + "name": "logits_soft_cap", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "0.0", + "annotation": "float" + }, + { + "name": "zero_tensors", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "cu_seqlens_q_padded", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "cu_seqlens_k_padded", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "s_aux", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "kwargs", + "kind": "VAR_KEYWORD", + "default": null, + "annotation": null + } + ], + "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]" + } + }, + { + "name": "flash_attn_with_kvcache", + "qualname": "flash_attn_with_kvcache", + "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha", + "kind": "function", + "doc": "FlashAttention against a KV cache, built on the CK split-KV kernel.\n\n``q`` is ``(batch, seqlen_q, nheads, headdim)``. For a contiguous cache,\n``k_cache``/``v_cache`` are ``(batch, seqlen_cache, nheads_k, headdim)`` and\n``cache_seqlens`` (int or ``(batch,)``) is the number of valid cache entries\n*before* appending. If ``k``/``v`` (the new tokens) are given, they are\nwritten into the cache in place. The query attends to the valid prefix\n(``cache_seqlens`` + ``seqlen_q``) with bottom-right causal alignment.\n\nNotes / current limitations:\n- Rotary embedding is not applied here; pass ``q``/``k`` already rotated\n (``rotary_cos``/``rotary_sin`` are rejected).\n- Paged caches are addressed via ``block_table`` (forwarded to the split-KV\n varlen kernel); in-cache append for paged layouts must be done by the\n caller (pass ``k=v=None``).", + "signature": { + "parameters": [ + { + "name": "q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "k_cache", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "v_cache", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "v", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "rotary_cos", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "rotary_sin", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "cache_seqlens", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": null + }, + { + "name": "block_table", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "softmax_scale", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "float | None" + }, + { + "name": "causal", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "True", + "annotation": "bool" + }, + { + "name": "window_size", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "(-1, -1, 0)", + "annotation": "Tuple[int, ...]" + }, + { + "name": "s_aux", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "return_lse", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "kwargs", + "kind": "VAR_KEYWORD", + "default": null, + "annotation": null + } + ], + "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]" + } + } + ], + "layers": [] +} diff --git a/build/torch215-rocm714-x86_64-linux/__init__.py b/build/torch215-rocm714-x86_64-linux/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..73acb58c1142a1ab2a6ed9d6f34bc6a53ab5e9b5 --- /dev/null +++ b/build/torch215-rocm714-x86_64-linux/__init__.py @@ -0,0 +1,27 @@ +"""AITER Composable-Kernel FlashAttention kernels for AMD ROCm (compiled HIP). + +Compiled (Composable Kernel / HIP) counterpart to the Triton-based +``aiter-flash-attn`` package, repackaged from the CK FMHA forward path of the +`ROCm/aiter `_ project as a self-contained +Hugging Face Hub kernel. Exposes the FlashAttention forward entry points with +learnable attention sink support (e.g. gpt-oss). +""" + +from .mha import ( + flash_attn_func, + flash_attn_varlen_func, + flash_attn_with_kvcache, +) + + +__kernel_metadata__ = { + "license": "mit", +} + + +__all__ = [ + "__kernel_metadata__", + "flash_attn_func", + "flash_attn_varlen_func", + "flash_attn_with_kvcache", +] diff --git a/build/torch215-rocm714-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so b/build/torch215-rocm714-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so new file mode 100644 index 0000000000000000000000000000000000000000..ac4aaf93c4fc39518afeb9030ee75e7bd263a61a --- /dev/null +++ b/build/torch215-rocm714-x86_64-linux/_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b99d985769f4ec0b35a776d3232576db28402060e79bf64cada77851f24d9d7 +size 159738320 diff --git a/build/torch215-rocm714-x86_64-linux/_ops.py b/build/torch215-rocm714-x86_64-linux/_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..2b4dd56c220b61fd2c16e0c1a02ee91a491cef0c --- /dev/null +++ b/build/torch215-rocm714-x86_64-linux/_ops.py @@ -0,0 +1,9 @@ +import torch +from . import _aiter_flash_attn_ck_rocm_fe2bb66 +ops = torch.ops._aiter_flash_attn_ck_rocm_fe2bb66 + +def add_op_namespace_prefix(op_name: str): + """ + Prefix op by namespace. + """ + return f"_aiter_flash_attn_ck_rocm_fe2bb66::{op_name}" diff --git a/build/torch215-rocm714-x86_64-linux/metadata.json b/build/torch215-rocm714-x86_64-linux/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..da34e4d511724c93ab4e3cd7318cf99c65a4afa0 --- /dev/null +++ b/build/torch215-rocm714-x86_64-linux/metadata.json @@ -0,0 +1,37 @@ +{ + "name": "aiter-flash-attn-ck", + "id": "_aiter_flash_attn_ck_rocm_fe2bb66", + "version": 1, + "kernels-minver": "0.14.0", + "license": "MIT", + "python-depends": [], + "kernel-depends": [], + "backend": { + "type": "rocm", + "archs": [ + "gfx942", + "gfx950" + ] + }, + "digest": { + "algorithm": "sha256", + "files": { + "__init__.py": "7wj1GTm2bqTLLCyT4xLuAk/+JWYrhINjY/ODMNWhgZ0=", + "_aiter_flash_attn_ck_rocm_fe2bb66.abi3.so": "S5nZhXafTsCzWndtMjJXbbKEAgYOeb9kytp3hR8k2dc=", + "_ops.py": "4WkdYDWWJp60zep+oW/5ym9vIeVMNfrgRCkP3o/w7yU=", + "mha.py": "zCAjY4O+DiCzmEL+W9sqXxZaqjsj4RJbWFcyPpUZejk=", + "symbols.json": "rhnF4H8Jsb6I+YPxIVFMgnLcTQJODEXbZ7ge77NZXAU=" + } + }, + "provenance": { + "kernel-builder": { + "version": "0.18.0-dev0", + "commit": "d054906a13710b811b96697b4635f03020842733", + "dirty": false + }, + "kernel": { + "commit": "fe2bb66fee6c2a77eccf08a0508dc57428d488ed", + "dirty": false + } + } +} \ No newline at end of file diff --git a/build/torch215-rocm714-x86_64-linux/metadata.json.sigstore b/build/torch215-rocm714-x86_64-linux/metadata.json.sigstore new file mode 100644 index 0000000000000000000000000000000000000000..85dcc12478e3403ab5667f95a9457843814c1e02 --- /dev/null +++ b/build/torch215-rocm714-x86_64-linux/metadata.json.sigstore @@ -0,0 +1 @@ +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json", "verificationMaterial":{"certificate":{"rawBytes":"MIIHLTCCBrOgAwIBAgIUPC6CLEODxN7FXbU5U4C8Y1TNhO0wCgYIKoZIzj0EAwMwNzEVMBMGA1UEChMMc2lnc3RvcmUuZGV2MR4wHAYDVQQDExVzaWdzdG9yZS1pbnRlcm1lZGlhdGUwHhcNMjYxMDE5MjIzMTI2WhcNMjYxMDE5MjI0MTI2WjAAMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEyG3/PH/4grzUMxbfXE8uvLU8YdwM6fY5bsg2HZWkh9ef2V55VLVU4o33o13cY/kCHNMWFiIQwQUEy0l7wR7qzaOCBdIwggXOMA4GA1UdDwEB/wQEAwIHgDATBgNVHSUEDDAKBggrBgEFBQcDAzAfBgNVHSMEGDAWgBTf0+nPViQRlvmo2OkoVaLGLhhkPzBrBgNVHREBAf8EYTBfhl1odHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkvLmdpdGh1Yi93b3JrZmxvd3MvYnVpbGQueWFtbEByZWZzL2hlYWRzL21haW4wOQYKKwYBBAGDvzABAQQraHR0cHM6Ly90b2tlbi5hY3Rpb25zLmdpdGh1YnVzZXJjb250ZW50LmNvbTAfBgorBgEEAYO/MAECBBF3b3JrZmxvd19kaXNwYXRjaDA2BgorBgEEAYO/MAEDBChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMBMGCisGAQQBg78wAQQEBUJ1aWxkMCsGCisGAQQBg78wAQUEHWh1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5MB0GCisGAQQBg78wAQYED3JlZnMvaGVhZHMvbWFpbjA7BgorBgEEAYO/MAEIBC0MK2h0dHBzOi8vdG9rZW4uYWN0aW9ucy5naXRodWJ1c2VyY29udGVudC5jb20wbQYKKwYBBAGDvzABCQRfDF1odHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkvLmdpdGh1Yi93b3JrZmxvd3MvYnVpbGQueWFtbEByZWZzL2hlYWRzL21haW4wOAYKKwYBBAGDvzABCgQqDChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMBsGCisGAQQBg78wAQsEDQwLc2VsZi1ob3N0ZWQwQAYKKwYBBAGDvzABDAQyDDBodHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHkwOAYKKwYBBAGDvzABDQQqDChmZTJiYjY2ZmVlNmMyYTc3ZWNjZjA4YTA1MDhkYzU3NDI4ZDQ4OGVkMB8GCisGAQQBg78wAQ4EEQwPcmVmcy9oZWFkcy9tYWluMBoGCisGAQQBg78wAQ8EDAwKMTA3MTQ3NTUyOTAuBgorBgEEAYO/MAEQBCAMHmh0dHBzOi8vZ2l0aHViLmNvbS9odWdnaW5nZmFjZTAYBgorBgEEAYO/MAERBAoMCDI1NzIwNzQzMG0GCisGAQQBg78wARIEXwxdaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5Ly5naXRodWIvd29ya2Zsb3dzL2J1aWxkLnlhbWxAcmVmcy9oZWFkcy9tYWluMDgGCisGAQQBg78wARMEKgwoZmUyYmI2NmZlZTZjMmE3N2VjY2YwOGEwNTA4ZGM1NzQyOGQ0ODhlZDAhBgorBgEEAYO/MAEUBBMMEXdvcmtmbG93X2Rpc3BhdGNoMGQGCisGAQQBg78wARUEVgxUaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5L2FjdGlvbnMvcnVucy8zOTIzOTUxODEwNC9hdHRlbXB0cy8xMBYGCisGAQQBg78wARYECAwGcHVibGljMEYGCisGAQQBg78wARgEOAw2cmVwbzpodWdnaW5nZmFjZS9rZXJuZWxzLWNvbW11bml0eTpyZWY6cmVmcy9oZWFkcy9tYWluMIGLBgorBgEEAdZ5AgQCBH0EewB5AHcA3T0wasbHETJjGR4cmWc3AqJKXrjePK3/h4pygC8p7o4AAAGhVkqD0AAABAMASDBGAiEA+7tSAw12m1Vkz5qXysWHIsywCGy4+xUU5DYANdKmFKsCIQCLV3xeirrAIhV4OOmIt/IhdcxjvZWg/KmVIRj6McPbWjAKBggqhkjOPQQDAwNoADBlAjAmWZHev40VL4wHR47f83C25a9bjhzqeMkYEgt8Swz3AnvoTsOA9mEvXVxOkfjj6gcCMQCGOQz/ln6oUxNvgB8OaKVoYxs+BBeH+buTCvxq60bWp21zRBH+7LMt4qlPqsdLA10="}, "tlogEntries":[{"logIndex":"3327213458", "logId":{"keyId":"wNI9atQGlz+VWfO6LRygH4QUfY/8W4RFwiT5i5WRgB0="}, "kindVersion":{"kind":"hashedrekord", "version":"0.0.1"}, "integratedTime":"1792449086", "inclusionPromise":{"signedEntryTimestamp":"MEYCIQCqp1C1H29TYHOYDf13FMNeXAa5zTRrAG9OyTw2yej9hwIhAJ2ncw8lCZOvje+B75jZVuvXCobSWMDHpHF9phzyA4Uv"}, "inclusionProof":{"logIndex":"3205309196", "rootHash":"d5Qbu9K1TnU4qYSbF6kpeZaiDx1zvVkf0b14AGm66eI=", "treeSize":"3205309278", "hashes":["hdZOgJ1ejEWnvOh0r0/mL8GhBLsE1hYOiM1PHXLdcUw=", "9r6ubahMpwvqlUP7gPzrYAWCVJX5YD0dMm2Z8s0LBL4=", "aO/UbKPcM1P5QMp+u56Lb3OQ/oNOGqBqpr7WF3YEoPo=", "wLijkhcDEwiFjuJLIoeXVB59httOt4PdmEuHly2JDgw=", "NCp7NxcSzc6sJbMMLAIYnvgi1MkcfJhxYynJojetRag=", "vtzzYd2oHkX9igPGkdBrqYOZIsGupEuRteHiRZY6KhY=", "+p6Oe75BmNRD+dmoE4KKGZkW+ikNVSIBCLGQotgHFJg=", "HYlQ7LKQ/ZSHqWPmKp61mcnEClxJ97KnMS47/s1UQoA=", "Wi2RFS8GCGJbHzk33Tu5qByYX+cjOpcYC+Xv4OfLeEI=", "1xxYV6+TE9ji17A5a8fHUMJY41tBhqzb3fUw9AcY6/M=", "QIMkwQbwAewhLJd//5FFwI3SnyNQBGjCgJrelfG8vhc=", "Et17CACUv9QN64irUJuBjsrgRGkGnnJ9xacRV/aBdwQ=", "lqgEwkxhDMs/gdIAE4lmjfMwg7nDmv7lda6O0LGliCw=", "HovTPBWzoErvUg1u5sIi3x1ZonZ7yJekVsBrmpomCIA=", "G+nk7mgE5h6dUmv2UY7NDJrY3WvrZY31raaO1AETt+o=", "qlTWNDzERnTBQVcHHXXVPjiUP0SsikGxfcxVlgqVFDo=", "gg9GNApNke3+zNhl4tsWtmIycCFrOMCB3lys6XuSQNQ=", "NlOclldmKPA1tMKIskqHAwuQDYHp31YwpK+OOMrMccc=", "qxzHanAzz57SDdmJe0B7bJK72NTIBbwEMGBBKvDOROw=", "xH/DCseLHr9eKoYT8qsORZK7zVdEGYWHuVtsVrD95wY="], "checkpoint":{"envelope":"rekor.sigstore.dev - 1193050959916656506\n3205309278\nd5Qbu9K1TnU4qYSbF6kpeZaiDx1zvVkf0b14AGm66eI=\n\n— rekor.sigstore.dev wNI9ajBEAiBeI+0twcEzIJOTlEGVhZO2nfzl/RTVtYbNQNh1TIZ0YAIgdSnO3SvE+wgTQ66B9cCTQ7aSVyNWVCgzVCsjsVTZvro=\n"}}, "canonicalizedBody":"eyJhcGlWZXJzaW9uIjoiMC4wLjEiLCJraW5kIjoiaGFzaGVkcmVrb3JkIiwic3BlYyI6eyJkYXRhIjp7Imhhc2giOnsiYWxnb3JpdGhtIjoic2hhMjU2IiwidmFsdWUiOiJlNjQ1MTVkYTEwYmRlNjAwNmRiOWExODhkZjA4MGQ3OTMxMjljY2U0ZWZkZWYyYzRhNTMxMjI4ZTgwMTQ0OTQyIn19LCJzaWduYXR1cmUiOnsiY29udGVudCI6Ik1FWUNJUUN1UFhUMkplZmx6alZIN1M3QXpKRlJCMW9Kc3I2YlJOU2lHdWtaYXN2NFZ3SWhBTytQbGI1L2xjN2RFSWU5MCtjbnlLSjMrV0pZTkVFZGIvNzEvL09UY1oveSIsInB1YmxpY0tleSI6eyJjb250ZW50IjoiTFMwdExTMUNSVWRKVGlCRFJWSlVTVVpKUTBGVVJTMHRMUzB0Q2sxSlNVaE1WRU5EUW5KUFowRjNTVUpCWjBsVlVFTTJRMHhGVDBSNFRqZEdXR0pWTlZVMFF6aFpNVlJPYUU4d2QwTm5XVWxMYjFwSmVtb3dSVUYzVFhjS1RucEZWazFDVFVkQk1WVkZRMmhOVFdNeWJHNWpNMUoyWTIxVmRWcEhWakpOVWpSM1NFRlpSRlpSVVVSRmVGWjZZVmRrZW1SSE9YbGFVekZ3WW01U2JBcGpiVEZzV2tkc2FHUkhWWGRJYUdOT1RXcFplRTFFUlRWTmFrbDZUVlJKTWxkb1kwNU5hbGw0VFVSRk5VMXFTVEJOVkVreVYycEJRVTFHYTNkRmQxbElDa3R2V2tsNmFqQkRRVkZaU1V0dldrbDZhakJFUVZGalJGRm5RVVY1UnpNdlVFZ3ZOR2R5ZWxWTmVHSm1XRVU0ZFhaTVZUaFpaSGROTm1aWk5XSnpaeklLU0ZwWGEyZzVaV1l5VmpVMVZreFdWVFJ2TXpOdk1UTmpXUzlyUTBoT1RWZEdhVWxSZDFGVlJYa3diRGQzVWpkeGVtRlBRMEprU1hkbloxaFBUVUUwUndwQk1WVmtSSGRGUWk5M1VVVkJkMGxJWjBSQlZFSm5UbFpJVTFWRlJFUkJTMEpuWjNKQ1owVkdRbEZqUkVGNlFXWkNaMDVXU0ZOTlJVZEVRVmRuUWxSbUNqQXJibEJXYVZGU2JIWnRiekpQYTI5V1lVeEhUR2hvYTFCNlFuSkNaMDVXU0ZKRlFrRm1PRVZaVkVKbWFHd3hiMlJJVW5kamVtOTJUREprY0dSSGFERUtXV2sxYW1JeU1IWmhTRlp1V2pKc2RWb3lXbWhaTWxWMllUSldlV0p0Vm5OamVURnFZakl4ZEdSWE5YQmtTR3QyVEcxa2NHUkhhREZaYVRrellqTktjZ3BhYlhoMlpETk5kbGx1Vm5CaVIxRjFaVmRHZEdKRlFubGFWMXA2VERKb2JGbFhVbnBNTWpGb1lWYzBkMDlSV1V0TGQxbENRa0ZIUkhaNlFVSkJVVkZ5Q21GSVVqQmpTRTAyVEhrNU1HSXlkR3hpYVRWb1dUTlNjR0l5TlhwTWJXUndaRWRvTVZsdVZucGFXRXBxWWpJMU1GcFhOVEJNYlU1MllsUkJaa0puYjNJS1FtZEZSVUZaVHk5TlFVVkRRa0pHTTJJelNuSmFiWGgyWkRFNWEyRllUbmRaV0ZKcVlVUkJNa0puYjNKQ1owVkZRVmxQTDAxQlJVUkNRMmh0V2xSS2FRcFphbGt5V20xV2JFNXRUWGxaVkdNeldsZE9hbHBxUVRSWlZFRXhUVVJvYTFsNlZUTk9SRWswV2tSUk5FOUhWbXROUWsxSFEybHpSMEZSVVVKbk56aDNDa0ZSVVVWQ1ZVb3hZVmQ0YTAxRGMwZERhWE5IUVZGUlFtYzNPSGRCVVZWRlNGZG9NVm95WkhCaWJXUnRXVmRPYkV3eWRHeGpiVFZzWWtoTmRGa3lPWFFLWWxoV2RXRllValZOUWpCSFEybHpSMEZSVVVKbk56aDNRVkZaUlVRelNteGFiazEyWVVkV2FGcElUWFppVjBad1ltcEJOMEpuYjNKQ1owVkZRVmxQTHdwTlFVVkpRa013VFVzeWFEQmtTRUo2VDJrNGRtUkhPWEphVnpSMVdWZE9NR0ZYT1hWamVUVnVZVmhTYjJSWFNqRmpNbFo1V1RJNWRXUkhWblZrUXpWcUNtSXlNSGRpVVZsTFMzZFpRa0pCUjBSMmVrRkNRMUZTWmtSR01XOWtTRkozWTNwdmRrd3laSEJrUjJneFdXazFhbUl5TUhaaFNGWnVXakpzZFZveVdtZ0tXVEpWZG1FeVZubGliVlp6WTNreGFtSXlNWFJrVnpWd1pFaHJka3h0WkhCa1IyZ3hXV2s1TTJJelNuSmFiWGgyWkROTmRsbHVWbkJpUjFGMVpWZEdkQXBpUlVKNVdsZGFla3d5YUd4WlYxSjZUREl4YUdGWE5IZFBRVmxMUzNkWlFrSkJSMFIyZWtGQ1EyZFJjVVJEYUcxYVZFcHBXV3BaTWxwdFZteE9iVTE1Q2xsVVl6TmFWMDVxV21wQk5GbFVRVEZOUkdocldYcFZNMDVFU1RSYVJGRTBUMGRXYTAxQ2MwZERhWE5IUVZGUlFtYzNPSGRCVVhORlJGRjNUR015Vm5NS1dta3hiMkl6VGpCYVYxRjNVVUZaUzB0M1dVSkNRVWRFZG5wQlFrUkJVWGxFUkVKdlpFaFNkMk42YjNaTU1tUndaRWRvTVZscE5XcGlNakIyWVVoV2JncGFNbXgxV2pKYWFGa3lWWFpoTWxaNVltMVdjMk41TVdwaU1qRjBaRmMxY0dSSWEzZFBRVmxMUzNkWlFrSkJSMFIyZWtGQ1JGRlJjVVJEYUcxYVZFcHBDbGxxV1RKYWJWWnNUbTFOZVZsVVl6TmFWMDVxV21wQk5GbFVRVEZOUkdocldYcFZNMDVFU1RSYVJGRTBUMGRXYTAxQ09FZERhWE5IUVZGUlFtYzNPSGNLUVZFMFJVVlJkMUJqYlZadFkzazViMXBYUm10amVUbDBXVmRzZFUxQ2IwZERhWE5IUVZGUlFtYzNPSGRCVVRoRlJFRjNTMDFVUVROTlZGRXpUbFJWZVFwUFZFRjFRbWR2Y2tKblJVVkJXVTh2VFVGRlVVSkRRVTFJYldnd1pFaENlazlwT0haYU1td3dZVWhXYVV4dFRuWmlVemx2WkZka2JtRlhOVzVhYlVacUNscFVRVmxDWjI5eVFtZEZSVUZaVHk5TlFVVlNRa0Z2VFVORVNURk9la2wzVG5wUmVrMUhNRWREYVhOSFFWRlJRbWMzT0hkQlVrbEZXSGQ0WkdGSVVqQUtZMGhOTmt4NU9XNWhXRkp2WkZkSmRWa3lPWFJNTW1neFdqSmtjR0p0WkcxWlYwNXNUREowYkdOdE5XeGlTRTEwV1RJNWRHSllWblZoV0ZJMVRIazFiZ3BoV0ZKdlpGZEpkbVF5T1hsaE1scHpZak5rZWt3eVNqRmhWM2hyVEc1c2FHSlhlRUZqYlZadFkzazViMXBYUm10amVUbDBXVmRzZFUxRVowZERhWE5IQ2tGUlVVSm5OemgzUVZKTlJVdG5kMjlhYlZWNVdXMUpNazV0V214YVZGcHFUVzFGTTA0eVZtcFpNbGwzVDBkRmQwNVVRVFJhUjAweFRucFJlVTlIVVRBS1QwUm9iRnBFUVdoQ1oyOXlRbWRGUlVGWlR5OU5RVVZWUWtKTlRVVllaSFpqYlhSdFlrYzVNMWd5VW5Cak0wSm9aRWRPYjAxSFVVZERhWE5IUVZGUlFncG5OemgzUVZKVlJWWm5lRlZoU0ZJd1kwaE5Oa3g1T1c1aFdGSnZaRmRKZFZreU9YUk1NbWd4V2pKa2NHSnRaRzFaVjA1c1RESjBiR050Tld4aVNFMTBDbGt5T1hSaVdGWjFZVmhTTlV3eVJtcGtSMngyWW01TmRtTnVWblZqZVRoNlQxUkplazlVVlhoUFJFVjNUa001YUdSSVVteGlXRUl3WTNrNGVFMUNXVWNLUTJselIwRlJVVUpuTnpoM1FWSlpSVU5CZDBkalNGWnBZa2RzYWsxRldVZERhWE5IUVZGUlFtYzNPSGRCVW1kRlQwRjNNbU50Vm5kaWVuQnZaRmRrYmdwaFZ6VnVXbTFHYWxwVE9YSmFXRXAxV2xkNGVreFhUblppVnpFeFltMXNNR1ZVY0hsYVYxazJZMjFXYldONU9XOWFWMFpyWTNrNWRGbFhiSFZOU1VkTUNrSm5iM0pDWjBWRlFXUmFOVUZuVVVOQ1NEQkZaWGRDTlVGSVkwRXpWREIzWVhOaVNFVlVTbXBIVWpSamJWZGpNMEZ4U2t0WWNtcGxVRXN6TDJnMGNIa0taME00Y0Rkdk5FRkJRVWRvVm10eFJEQkJRVUZDUVUxQlUwUkNSMEZwUlVFck4zUlRRWGN4TW0weFZtdDZOWEZZZVhOWFNFbHplWGREUjNrMEszaFZWUW8xUkZsQlRtUkxiVVpMYzBOSlVVTk1Wak40WldseWNrRkphRlkwVDA5dFNYUXZTV2hrWTNocWRscFhaeTlMYlZaSlVtbzJUV05RWWxkcVFVdENaMmR4Q21ocmFrOVFVVkZFUVhkT2IwRkVRbXhCYWtGdFYxcElaWFkwTUZaTU5IZElValEzWmpnelF6STFZVGxpYW1oNmNXVk5hMWxGWjNRNFUzZDZNMEZ1ZG04S1ZITlBRVGx0UlhaWVZuaFBhMlpxYWpablkwTk5VVU5IVDFGNkwyeHVObTlWZUU1MlowSTRUMkZMVm05WmVITXJRa0psU0N0aWRWUkRkbmh4TmpCaVZ3cHdNakY2VWtKSUt6ZE1UWFEwY1d4UWNYTmtURUV4TUQwS0xTMHRMUzFGVGtRZ1EwVlNWRWxHU1VOQlZFVXRMUzB0TFFvPSJ9fX19"}], "timestampVerificationData":{"rfc3161Timestamps":[{"signedTimestamp":"MIICyDADAgEAMIICvwYJKoZIhvcNAQcCoIICsDCCAqwCAQMxDTALBglghkgBZQMEAgEwgbcGCyqGSIb3DQEJEAEEoIGnBIGkMIGhAgEBBgkrBgEEAYO/MAIwMTANBglghkgBZQMEAgEFAAQgZ5lAujbHHXDPViq4NyCExMOQGQdZv+ozUWdO1I2rzvUCFAXz22ii7bs3euDJeX7cqcx2kyxhGA8yMDI2MTAxOTIyMzEyNlowAwIBAaAypDAwLjEVMBMGA1UEChMMc2lnc3RvcmUuZGV2MRUwEwYDVQQDEwxzaWdzdG9yZS10c2GgADGCAdowggHWAgEBMFEwOTEVMBMGA1UEChMMc2lnc3RvcmUuZGV2MSAwHgYDVQQDExdzaWdzdG9yZS10c2Etc2VsZnNpZ25lZAIUOhNULwyQYe68wUMvy4qOiyojiwwwCwYJYIZIAWUDBAIBoIH8MBoGCSqGSIb3DQEJAzENBgsqhkiG9w0BCRABBDAcBgkqhkiG9w0BCQUxDxcNMjYxMDE5MjIzMTI2WjAvBgkqhkiG9w0BCQQxIgQgLVqaBESXIdO/G1nERph0R387d66Bi0kDGLgueP1yEAMwgY4GCyqGSIb3DQEJEAIvMX8wfTB7MHkEIIX5J7wHq2LKw7RDVsEO/IGyxog/2nq55thw2dE6zQW3MFUwPaQ7MDkxFTATBgNVBAoTDHNpZ3N0b3JlLmRldjEgMB4GA1UEAxMXc2lnc3RvcmUtdHNhLXNlbGZzaWduZWQCFDoTVC8MkGHuvMFDL8uKjosqI4sMMAoGCCqGSM49BAMCBGYwZAIwGV+5PRiXIpOBm75ilz4qd/iMEfdTyuikBKXyNVrOF5m82wEXe/FRZw5hOIxjME8DAjA9Hce+C/iFEkXHHET8JmtOtTCe6WZW16M4cpKy6+4pO6BH0S4x8q9+ZyFCo6ysVME="}]}}, "messageSignature":{"messageDigest":{"algorithm":"SHA2_256", "digest":"5kUV2hC95gBtuaGI3wgNeTEpzOTv3vLEpTEijoAUSUI="}, "signature":"MEYCIQCuPXT2JeflzjVH7S7AzJFRB1oJsr6bRNSiGukZasv4VwIhAO+Plb5/lc7dEIe90+cnyKJ3+WJYNEEdb/71//OTcZ/y"}} \ No newline at end of file diff --git a/build/torch215-rocm714-x86_64-linux/mha.py b/build/torch215-rocm714-x86_64-linux/mha.py new file mode 100644 index 0000000000000000000000000000000000000000..01e2d6a0fbee98d27129f2eb695ac38df73e6487 --- /dev/null +++ b/build/torch215-rocm714-x86_64-linux/mha.py @@ -0,0 +1,332 @@ +"""FlashAttention forward entry points backed by aiter's Composable-Kernel FMHA. + +This is the compiled (HIP / Composable Kernel) counterpart to the Triton +``aiter-flash-attn`` package. Only the *forward* pass is provided (dense and +variable-length), including support for learnable attention sinks via +``window_size[2]`` / ``s_aux`` (e.g. gpt-oss). The op set ships exclusively +the CK ``mha_fwd`` / ``mha_varlen_fwd`` kernels, so the sink path is always +taken (no fall-through to a non-sink ASM kernel). +""" + +from typing import Optional, Tuple, Union + +import torch +from torch import Tensor + +from ._ops import ops + + +def _round_multiple(x: int, m: int) -> int: + return (x + m - 1) // m * m + + +def _maybe_pad_headdim(t: Tensor, target: int) -> Tensor: + og = t.size(-1) + if og == target: + return t + return torch.nn.functional.pad(t, [0, target - og]) + + +def _unpack_window(window_size) -> Tuple[int, int, int]: + left = int(window_size[0]) + right = int(window_size[1]) + sink = int(window_size[2]) if len(window_size) == 3 else 0 + return left, right, sink + + +def _prep_sink(s_aux: Optional[Tensor], nheads: int, device) -> Optional[Tensor]: + if s_aux is None: + return None + assert s_aux.device == device, "s_aux must be on the same device as q" + assert s_aux.shape[0] == nheads, "s_aux must have shape [nheads_q]" + if s_aux.dtype != torch.float32: + s_aux = s_aux.to(torch.float32) + return s_aux + + +def _effective_sink_size(sink_size: int, s_aux: Optional[Tensor]) -> int: + # The CK kernel only takes the sink path when sink_size > 0. Callers using + # the transformers attention interface pass the per-head sink logits as + # ``s_aux`` but leave ``window_size`` a 2-tuple (sink_size == 0), so enable + # a single learnable sink automatically when s_aux is given. + if s_aux is not None and sink_size == 0: + return 1 + return sink_size + + +def flash_attn_func( + q: Tensor, + k: Tensor, + v: Tensor, + dropout_p: float = 0.0, + softmax_scale: Optional[float] = None, + causal: bool = False, + window_size: Tuple[int, ...] = (-1, -1, 0), + bias: Optional[Tensor] = None, + alibi_slopes: Optional[Tensor] = None, + return_lse: bool = False, + return_attn_probs: bool = False, + s_aux: Optional[Tensor] = None, + **kwargs, +) -> Union[Tensor, Tuple[Tensor, ...]]: + """Dense FlashAttention forward. + + ``q, k, v`` have shape ``(batch, seqlen, nheads, headdim)``. ``window_size`` + is ``(left, right, sink_size)``; ``s_aux`` is an optional ``(nheads_q,)`` + fp32 tensor of learnable sink logits. Extra keyword arguments (e.g. those + passed by the transformers flash-attention interface) are ignored. + """ + if softmax_scale is None: + softmax_scale = q.shape[-1] ** (-0.5) + window_size_left, window_size_right, sink_size = _unpack_window(window_size) + + nheads_q = q.size(2) + s_aux = _prep_sink(s_aux, nheads_q, q.device) + sink_size = _effective_sink_size(sink_size, s_aux) + + head_q_og = q.size(-1) + head_v_og = v.size(-1) + head_q = _round_multiple(head_q_og, 8) + head_v = _round_multiple(head_v_og, 8) + q = _maybe_pad_headdim(q, head_q) + k = _maybe_pad_headdim(k, head_q) + v = _maybe_pad_headdim(v, head_v) + + return_softmax = return_attn_probs and dropout_p > 0 + out, softmax_lse, S_dmask, _ = ops.mha_fwd( + q, + k, + v, + dropout_p, + softmax_scale, + causal, + window_size_left, + window_size_right, + sink_size, + return_lse or return_attn_probs, + return_softmax, + None, # cu_seqlens_q + None, # cu_seqlens_kv + None, # out + bias, + alibi_slopes, + None, # q_descale + None, # k_descale + None, # v_descale + s_aux, + None, # gen + ) + + out = out[..., :head_v_og] + if not (return_lse or return_attn_probs): + return out + result = (out,) + if return_lse: + result = result + (softmax_lse,) + if return_attn_probs: + result = result + (S_dmask,) + return result + + +def flash_attn_varlen_func( + q: Tensor, + k: Tensor, + v: Tensor, + cu_seqlens_q: Tensor, + cu_seqlens_k: Tensor, + max_seqlen_q: int, + max_seqlen_k: int, + dropout_p: float = 0.0, + softmax_scale: Optional[float] = None, + causal: bool = False, + window_size: Tuple[int, ...] = (-1, -1, 0), + bias: Optional[Tensor] = None, + alibi_slopes: Optional[Tensor] = None, + return_lse: bool = False, + return_attn_probs: bool = False, + block_table: Optional[Tensor] = None, + min_seqlen_q: int = 0, + logits_soft_cap: float = 0.0, + zero_tensors: bool = False, + cu_seqlens_q_padded: Optional[Tensor] = None, + cu_seqlens_k_padded: Optional[Tensor] = None, + s_aux: Optional[Tensor] = None, + **kwargs, +) -> Union[Tensor, Tuple[Tensor, ...]]: + """Variable-length / packed FlashAttention forward. + + ``q`` has shape ``(total_q, nheads, headdim)``; ``cu_seqlens_*`` are int32 + cumulative offsets of shape ``(batch + 1,)``. Sinks are supported through + ``window_size[2]`` / ``s_aux`` as in :func:`flash_attn_func`. Extra keyword + arguments (e.g. from the transformers flash-attention interface) are ignored. + """ + if softmax_scale is None: + softmax_scale = q.shape[-1] ** (-0.5) + window_size_left, window_size_right, sink_size = _unpack_window(window_size) + + nheads_q = q.size(1) + s_aux = _prep_sink(s_aux, nheads_q, q.device) + sink_size = _effective_sink_size(sink_size, s_aux) + + head_q_og = q.size(-1) + head_v_og = v.size(-1) + head_q = _round_multiple(head_q_og, 8) + head_v = _round_multiple(head_v_og, 8) + q = _maybe_pad_headdim(q, head_q) + k = _maybe_pad_headdim(k, head_q) + v = _maybe_pad_headdim(v, head_v) + + return_softmax = return_attn_probs and dropout_p > 0 + out, softmax_lse, S_dmask, _ = ops.mha_varlen_fwd( + q, + k, + v, + cu_seqlens_q, + cu_seqlens_k, + max_seqlen_q, + max_seqlen_k, + min_seqlen_q, + dropout_p, + softmax_scale, + logits_soft_cap, + zero_tensors, + causal, + window_size_left, + window_size_right, + sink_size, + return_lse or return_attn_probs, + return_softmax, + None, # out + block_table, + bias, + alibi_slopes, + None, # q_descale + None, # k_descale + None, # v_descale + None, # gen + cu_seqlens_q_padded, + cu_seqlens_k_padded, + s_aux, + ) + + out = out[..., :head_v_og] + if not (return_lse or return_attn_probs): + return out + result = (out,) + if return_lse: + result = result + (softmax_lse,) + if return_attn_probs: + result = result + (S_dmask,) + return result + + +def _append_to_contiguous_cache(k_cache, v_cache, k, v, cache_seqlens, Sq): + """In-place append of new k/v into a contiguous cache at per-batch offsets.""" + B = k_cache.shape[0] + if isinstance(cache_seqlens, int): + cache_seqlens = torch.full((B,), cache_seqlens, device=k_cache.device, dtype=torch.long) + pos = cache_seqlens.view(B, 1).to(torch.long) + torch.arange(Sq, device=k_cache.device).view(1, Sq) + bidx = torch.arange(B, device=k_cache.device).view(B, 1).expand(B, Sq) + k_cache[bidx, pos] = k.to(k_cache.dtype) + v_cache[bidx, pos] = v.to(v_cache.dtype) + + +def flash_attn_with_kvcache( + q: Tensor, + k_cache: Tensor, + v_cache: Tensor, + k: Optional[Tensor] = None, + v: Optional[Tensor] = None, + rotary_cos: Optional[Tensor] = None, + rotary_sin: Optional[Tensor] = None, + cache_seqlens=None, + block_table: Optional[Tensor] = None, + softmax_scale: Optional[float] = None, + causal: bool = True, + window_size: Tuple[int, ...] = (-1, -1, 0), + s_aux: Optional[Tensor] = None, + return_lse: bool = False, + **kwargs, +) -> Union[Tensor, Tuple[Tensor, ...]]: + """FlashAttention against a KV cache, built on the CK split-KV kernel. + + ``q`` is ``(batch, seqlen_q, nheads, headdim)``. For a contiguous cache, + ``k_cache``/``v_cache`` are ``(batch, seqlen_cache, nheads_k, headdim)`` and + ``cache_seqlens`` (int or ``(batch,)``) is the number of valid cache entries + *before* appending. If ``k``/``v`` (the new tokens) are given, they are + written into the cache in place. The query attends to the valid prefix + (``cache_seqlens`` + ``seqlen_q``) with bottom-right causal alignment. + + Notes / current limitations: + - Rotary embedding is not applied here; pass ``q``/``k`` already rotated + (``rotary_cos``/``rotary_sin`` are rejected). + - Paged caches are addressed via ``block_table`` (forwarded to the split-KV + varlen kernel); in-cache append for paged layouts must be done by the + caller (pass ``k=v=None``). + """ + if rotary_cos is not None or rotary_sin is not None: + raise NotImplementedError( + "flash_attn_with_kvcache does not apply rotary; rotate q/k beforehand." + ) + if softmax_scale is None: + softmax_scale = q.shape[-1] ** (-0.5) + window_size_left, window_size_right, sink_size = _unpack_window(window_size) + B, Sq, Hq, D = q.shape + s_aux = _prep_sink(s_aux, Hq, q.device) + sink_size = _effective_sink_size(sink_size, s_aux) + + paged = block_table is not None + + # ---- append new tokens into the cache (contiguous layout only) ---- + if k is not None and not paged: + assert cache_seqlens is not None, "cache_seqlens required to append into the cache" + _append_to_contiguous_cache(k_cache, v_cache, k, v, cache_seqlens, Sq) + elif k is not None and paged: + raise NotImplementedError( + "in-cache append for paged layouts is not supported; append externally " + "and call with k=v=None." + ) + + appended = 0 if k is None else Sq + + # ---- key lengths ---- + if cache_seqlens is None: + klens = torch.full((B,), k_cache.shape[1], device=q.device, dtype=torch.long) + elif isinstance(cache_seqlens, int): + klens = torch.full((B,), cache_seqlens + appended, device=q.device, dtype=torch.long) + else: + klens = cache_seqlens.to(q.device, torch.long) + appended + + uniform = bool((klens == klens[0]).all().item()) + + # ---- fast path: contiguous cache, uniform length -> dense kernel ---- + if not paged and uniform: + L = int(klens[0].item()) + return flash_attn_func( + q, k_cache[:, :L], v_cache[:, :L], + softmax_scale=softmax_scale, causal=causal, + window_size=(window_size_left, window_size_right, sink_size), + s_aux=s_aux, return_lse=return_lse, + ) + + # ---- general path: variable cache lengths -> split-KV varlen kernel ---- + cu_q = torch.arange(0, (B + 1) * Sq, Sq, device=q.device, dtype=torch.int32) + cu_k = torch.zeros(B + 1, device=q.device, dtype=torch.int32) + cu_k[1:] = torch.cumsum(klens, 0).to(torch.int32) + q_packed = q.reshape(B * Sq, Hq, D) + if paged: + k_packed, v_packed = k_cache, v_cache # kernel gathers via block_table + else: + k_packed = torch.cat([k_cache[b, : int(klens[b])] for b in range(B)], dim=0) + v_packed = torch.cat([v_cache[b, : int(klens[b])] for b in range(B)], dim=0) + + out = flash_attn_varlen_func( + q_packed, k_packed, v_packed, cu_q, cu_k, Sq, int(klens.max().item()), + softmax_scale=softmax_scale, causal=causal, + window_size=(window_size_left, window_size_right, sink_size), + block_table=block_table, s_aux=s_aux, return_lse=return_lse, + ) + if return_lse: + out, lse = out + return out.reshape(B, Sq, Hq, D), lse + return out.reshape(B, Sq, Hq, D) diff --git a/build/torch215-rocm714-x86_64-linux/symbols.json b/build/torch215-rocm714-x86_64-linux/symbols.json new file mode 100644 index 0000000000000000000000000000000000000000..e8348db4bd94712352e7fb3e83b1d94f47c2814b --- /dev/null +++ b/build/torch215-rocm714-x86_64-linux/symbols.json @@ -0,0 +1,349 @@ +{ + "schema_version": 1, + "module": "_aiter_flash_attn_ck_rocm_fe2bb66", + "functions": [ + { + "name": "flash_attn_func", + "qualname": "flash_attn_func", + "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha", + "kind": "function", + "doc": "Dense FlashAttention forward.\n\n``q, k, v`` have shape ``(batch, seqlen, nheads, headdim)``. ``window_size``\nis ``(left, right, sink_size)``; ``s_aux`` is an optional ``(nheads_q,)``\nfp32 tensor of learnable sink logits. Extra keyword arguments (e.g. those\npassed by the transformers flash-attention interface) are ignored.", + "signature": { + "parameters": [ + { + "name": "q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "v", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "dropout_p", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "0.0", + "annotation": "float" + }, + { + "name": "softmax_scale", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "float | None" + }, + { + "name": "causal", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "window_size", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "(-1, -1, 0)", + "annotation": "Tuple[int, ...]" + }, + { + "name": "bias", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "alibi_slopes", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "return_lse", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "return_attn_probs", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "s_aux", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "kwargs", + "kind": "VAR_KEYWORD", + "default": null, + "annotation": null + } + ], + "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]" + } + }, + { + "name": "flash_attn_varlen_func", + "qualname": "flash_attn_varlen_func", + "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha", + "kind": "function", + "doc": "Variable-length / packed FlashAttention forward.\n\n``q`` has shape ``(total_q, nheads, headdim)``; ``cu_seqlens_*`` are int32\ncumulative offsets of shape ``(batch + 1,)``. Sinks are supported through\n``window_size[2]`` / ``s_aux`` as in :func:`flash_attn_func`. Extra keyword\narguments (e.g. from the transformers flash-attention interface) are ignored.", + "signature": { + "parameters": [ + { + "name": "q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "v", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "cu_seqlens_q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "cu_seqlens_k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "max_seqlen_q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "int" + }, + { + "name": "max_seqlen_k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "int" + }, + { + "name": "dropout_p", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "0.0", + "annotation": "float" + }, + { + "name": "softmax_scale", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "float | None" + }, + { + "name": "causal", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "window_size", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "(-1, -1, 0)", + "annotation": "Tuple[int, ...]" + }, + { + "name": "bias", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "alibi_slopes", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "return_lse", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "return_attn_probs", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "block_table", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "min_seqlen_q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "0", + "annotation": "int" + }, + { + "name": "logits_soft_cap", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "0.0", + "annotation": "float" + }, + { + "name": "zero_tensors", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "cu_seqlens_q_padded", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "cu_seqlens_k_padded", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "s_aux", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "kwargs", + "kind": "VAR_KEYWORD", + "default": null, + "annotation": null + } + ], + "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]" + } + }, + { + "name": "flash_attn_with_kvcache", + "qualname": "flash_attn_with_kvcache", + "module": "_aiter_flash_attn_ck_rocm_fe2bb66.mha", + "kind": "function", + "doc": "FlashAttention against a KV cache, built on the CK split-KV kernel.\n\n``q`` is ``(batch, seqlen_q, nheads, headdim)``. For a contiguous cache,\n``k_cache``/``v_cache`` are ``(batch, seqlen_cache, nheads_k, headdim)`` and\n``cache_seqlens`` (int or ``(batch,)``) is the number of valid cache entries\n*before* appending. If ``k``/``v`` (the new tokens) are given, they are\nwritten into the cache in place. The query attends to the valid prefix\n(``cache_seqlens`` + ``seqlen_q``) with bottom-right causal alignment.\n\nNotes / current limitations:\n- Rotary embedding is not applied here; pass ``q``/``k`` already rotated\n (``rotary_cos``/``rotary_sin`` are rejected).\n- Paged caches are addressed via ``block_table`` (forwarded to the split-KV\n varlen kernel); in-cache append for paged layouts must be done by the\n caller (pass ``k=v=None``).", + "signature": { + "parameters": [ + { + "name": "q", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "k_cache", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "v_cache", + "kind": "POSITIONAL_OR_KEYWORD", + "default": null, + "annotation": "torch.Tensor" + }, + { + "name": "k", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "v", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "rotary_cos", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "rotary_sin", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "cache_seqlens", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": null + }, + { + "name": "block_table", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "softmax_scale", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "float | None" + }, + { + "name": "causal", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "True", + "annotation": "bool" + }, + { + "name": "window_size", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "(-1, -1, 0)", + "annotation": "Tuple[int, ...]" + }, + { + "name": "s_aux", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "None", + "annotation": "torch.Tensor | None" + }, + { + "name": "return_lse", + "kind": "POSITIONAL_OR_KEYWORD", + "default": "False", + "annotation": "bool" + }, + { + "name": "kwargs", + "kind": "VAR_KEYWORD", + "default": null, + "annotation": null + } + ], + "return_annotation": "torch.Tensor | Tuple[torch.Tensor, ...]" + } + } + ], + "layers": [] +}